2024-11-08 02:30:21,634 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 2024-11-08 02:30:21,649 main DEBUG Took 0.013105 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 02:30:21,650 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 02:30:21,650 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 02:30:21,651 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 02:30:21,653 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,662 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 02:30:21,674 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,675 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,676 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,676 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,677 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,677 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,679 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,679 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,680 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,680 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,681 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,681 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,682 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,683 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,684 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,685 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,686 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,686 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,687 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 02:30:21,688 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,688 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 02:30:21,690 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 02:30:21,692 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 02:30:21,694 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 02:30:21,695 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 02:30:21,697 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 02:30:21,697 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 02:30:21,705 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 02:30:21,707 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 02:30:21,709 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 02:30:21,710 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 02:30:21,710 main DEBUG createAppenders(={Console}) 2024-11-08 02:30:21,711 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 initialized 2024-11-08 02:30:21,711 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 2024-11-08 02:30:21,712 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 OK. 2024-11-08 02:30:21,712 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 02:30:21,713 main DEBUG OutputStream closed 2024-11-08 02:30:21,713 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 02:30:21,713 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 02:30:21,714 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@54e1c68b OK 2024-11-08 02:30:21,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 02:30:21,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 02:30:21,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 02:30:21,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 02:30:21,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 02:30:21,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 02:30:21,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 02:30:21,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 02:30:21,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 02:30:21,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 02:30:21,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 02:30:21,808 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 02:30:21,808 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 02:30:21,809 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 02:30:21,809 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 02:30:21,810 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 02:30:21,810 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 02:30:21,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 02:30:21,813 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 02:30:21,813 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7dda48d9) with optional ClassLoader: null 2024-11-08 02:30:21,813 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 02:30:21,814 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@7dda48d9] started OK. 2024-11-08T02:30:22,036 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a 2024-11-08 02:30:22,038 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 02:30:22,038 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T02:30:22,046 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithBasicPolicy timeout: 13 mins 2024-11-08T02:30:22,065 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T02:30:22,068 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c, deleteOnExit=true 2024-11-08T02:30:22,069 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-08T02:30:22,069 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/test.cache.data in system properties and HBase conf 2024-11-08T02:30:22,070 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T02:30:22,071 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/hadoop.log.dir in system properties and HBase conf 2024-11-08T02:30:22,072 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T02:30:22,072 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T02:30:22,073 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-08T02:30:22,159 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T02:30:22,241 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T02:30:22,245 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T02:30:22,245 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T02:30:22,246 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T02:30:22,246 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T02:30:22,247 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T02:30:22,247 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T02:30:22,248 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T02:30:22,248 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T02:30:22,248 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T02:30:22,249 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/nfs.dump.dir in system properties and HBase conf 2024-11-08T02:30:22,249 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/java.io.tmpdir in system properties and HBase conf 2024-11-08T02:30:22,249 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T02:30:22,250 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T02:30:22,250 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T02:30:23,182 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T02:30:23,253 INFO [Time-limited test {}] log.Log(170): Logging initialized @2255ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T02:30:23,320 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T02:30:23,380 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T02:30:23,403 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T02:30:23,404 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T02:30:23,406 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T02:30:23,423 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T02:30:23,428 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8167a4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/hadoop.log.dir/,AVAILABLE} 2024-11-08T02:30:23,429 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62a9beb2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T02:30:23,639 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b03fcff{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/java.io.tmpdir/jetty-localhost-45067-hadoop-hdfs-3_4_1-tests_jar-_-any-10853162506341221826/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T02:30:23,648 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e0a3ea0{HTTP/1.1, (http/1.1)}{localhost:45067} 2024-11-08T02:30:23,648 INFO [Time-limited test {}] server.Server(415): Started @2651ms 2024-11-08T02:30:24,099 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T02:30:24,106 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T02:30:24,109 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T02:30:24,109 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T02:30:24,109 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T02:30:24,110 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47db50b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/hadoop.log.dir/,AVAILABLE} 2024-11-08T02:30:24,111 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4727fac8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T02:30:24,215 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1f79ec76{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/java.io.tmpdir/jetty-localhost-34651-hadoop-hdfs-3_4_1-tests_jar-_-any-4322904849223855718/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T02:30:24,215 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@576ebda6{HTTP/1.1, (http/1.1)}{localhost:34651} 2024-11-08T02:30:24,216 INFO [Time-limited test {}] server.Server(415): Started @3219ms 2024-11-08T02:30:24,263 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T02:30:24,991 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c/dfs/data/data1/current/BP-1683983252-172.17.0.2-1731033022817/current, will proceed with Du for space computation calculation, 2024-11-08T02:30:24,991 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c/dfs/data/data2/current/BP-1683983252-172.17.0.2-1731033022817/current, will proceed with Du for space computation calculation, 2024-11-08T02:30:25,017 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T02:30:25,061 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a 2024-11-08T02:30:25,064 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb0a9787c8d12518f with lease ID 0x7b352ac579d63b89: Processing first storage report for DS-486aa015-d11c-4a8a-a692-851a6f1d3552 from datanode DatanodeRegistration(127.0.0.1:39161, datanodeUuid=29f3b9ad-8562-4ec9-b455-cdc99a1c76ce, infoPort=43907, infoSecurePort=0, ipcPort=41835, storageInfo=lv=-57;cid=testClusterID;nsid=1215351802;c=1731033022817) 2024-11-08T02:30:25,066 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb0a9787c8d12518f with lease ID 0x7b352ac579d63b89: from storage DS-486aa015-d11c-4a8a-a692-851a6f1d3552 node DatanodeRegistration(127.0.0.1:39161, datanodeUuid=29f3b9ad-8562-4ec9-b455-cdc99a1c76ce, infoPort=43907, infoSecurePort=0, ipcPort=41835, storageInfo=lv=-57;cid=testClusterID;nsid=1215351802;c=1731033022817), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-08T02:30:25,066 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb0a9787c8d12518f with lease ID 0x7b352ac579d63b89: Processing first storage report for DS-9c349615-2835-441f-a7fd-b0d7efe8a6e0 from datanode DatanodeRegistration(127.0.0.1:39161, datanodeUuid=29f3b9ad-8562-4ec9-b455-cdc99a1c76ce, infoPort=43907, infoSecurePort=0, ipcPort=41835, storageInfo=lv=-57;cid=testClusterID;nsid=1215351802;c=1731033022817) 2024-11-08T02:30:25,067 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb0a9787c8d12518f with lease ID 0x7b352ac579d63b89: from storage DS-9c349615-2835-441f-a7fd-b0d7efe8a6e0 node DatanodeRegistration(127.0.0.1:39161, datanodeUuid=29f3b9ad-8562-4ec9-b455-cdc99a1c76ce, infoPort=43907, infoSecurePort=0, ipcPort=41835, storageInfo=lv=-57;cid=testClusterID;nsid=1215351802;c=1731033022817), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T02:30:25,128 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c/zookeeper_0, clientPort=56877, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T02:30:25,138 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=56877 2024-11-08T02:30:25,148 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:30:25,151 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:30:25,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741825_1001 (size=7) 2024-11-08T02:30:25,767 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 with version=8 2024-11-08T02:30:25,768 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/hbase-staging 2024-11-08T02:30:25,879 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T02:30:26,119 INFO [Time-limited test {}] client.ConnectionUtils(129): master/ecdbff65922a:0 server-side Connection retries=45 2024-11-08T02:30:26,135 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:30:26,135 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T02:30:26,136 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T02:30:26,136 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:30:26,136 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T02:30:26,252 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T02:30:26,300 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T02:30:26,308 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T02:30:26,311 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T02:30:26,334 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 46508 (auto-detected) 2024-11-08T02:30:26,335 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T02:30:26,351 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40841 2024-11-08T02:30:26,358 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:30:26,360 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:30:26,371 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:40841 connecting to ZooKeeper ensemble=127.0.0.1:56877 2024-11-08T02:30:26,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:408410x0, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T02:30:26,467 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40841-0x10119916ccb0000 connected 2024-11-08T02:30:26,538 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T02:30:26,541 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:30:26,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T02:30:26,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40841 2024-11-08T02:30:26,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40841 2024-11-08T02:30:26,550 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40841 2024-11-08T02:30:26,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40841 2024-11-08T02:30:26,551 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40841 2024-11-08T02:30:26,558 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643, hbase.cluster.distributed=false 2024-11-08T02:30:26,612 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/ecdbff65922a:0 server-side Connection retries=45 2024-11-08T02:30:26,612 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:30:26,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T02:30:26,613 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T02:30:26,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T02:30:26,613 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T02:30:26,615 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T02:30:26,617 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T02:30:26,618 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40577 2024-11-08T02:30:26,620 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T02:30:26,624 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T02:30:26,625 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:30:26,627 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:30:26,630 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:40577 connecting to ZooKeeper ensemble=127.0.0.1:56877 2024-11-08T02:30:26,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405770x0, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T02:30:26,775 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40577-0x10119916ccb0001 connected 2024-11-08T02:30:26,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T02:30:26,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:30:26,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T02:30:26,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40577 2024-11-08T02:30:26,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40577 2024-11-08T02:30:26,783 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40577 2024-11-08T02:30:26,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40577 2024-11-08T02:30:26,784 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40577 2024-11-08T02:30:26,786 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/ecdbff65922a,40841,1731033025873 2024-11-08T02:30:26,805 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ecdbff65922a:40841 2024-11-08T02:30:26,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:30:26,810 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:30:26,812 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ecdbff65922a,40841,1731033025873 2024-11-08T02:30:26,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T02:30:26,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T02:30:26,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:26,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:26,837 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T02:30:26,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T02:30:26,838 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ecdbff65922a,40841,1731033025873 from backup master directory 2024-11-08T02:30:26,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ecdbff65922a,40841,1731033025873 2024-11-08T02:30:26,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:30:26,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T02:30:26,845 WARN [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T02:30:26,845 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ecdbff65922a,40841,1731033025873 2024-11-08T02:30:26,848 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T02:30:26,850 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T02:30:26,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741826_1002 (size=42) 2024-11-08T02:30:27,321 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/hbase.id with ID: 8df15f50-b0f4-40bc-ac8d-430988525893 2024-11-08T02:30:27,364 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T02:30:27,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:27,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:27,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741827_1003 (size=196) 2024-11-08T02:30:27,869 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:30:27,870 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T02:30:27,883 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:27,886 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T02:30:27,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741828_1004 (size=1189) 2024-11-08T02:30:28,330 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store 2024-11-08T02:30:28,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741829_1005 (size=34) 2024-11-08T02:30:28,755 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T02:30:28,755 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:28,756 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T02:30:28,757 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:30:28,757 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:30:28,757 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T02:30:28,757 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:30:28,757 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:30:28,757 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T02:30:28,759 WARN [master/ecdbff65922a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/.initializing 2024-11-08T02:30:28,759 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/WALs/ecdbff65922a,40841,1731033025873 2024-11-08T02:30:28,765 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T02:30:28,775 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ecdbff65922a%2C40841%2C1731033025873, suffix=, logDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/WALs/ecdbff65922a,40841,1731033025873, archiveDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/oldWALs, maxLogs=10 2024-11-08T02:30:28,793 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/WALs/ecdbff65922a,40841,1731033025873/ecdbff65922a%2C40841%2C1731033025873.1731033028778, exclude list is [], retry=0 2024-11-08T02:30:28,807 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39161,DS-486aa015-d11c-4a8a-a692-851a6f1d3552,DISK] 2024-11-08T02:30:28,810 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T02:30:28,842 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/WALs/ecdbff65922a,40841,1731033025873/ecdbff65922a%2C40841%2C1731033025873.1731033028778 2024-11-08T02:30:28,843 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43907:43907)] 2024-11-08T02:30:28,843 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:30:28,844 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:28,847 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,848 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,901 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T02:30:28,904 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:28,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:30:28,907 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,910 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T02:30:28,911 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:28,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:30:28,912 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,915 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T02:30:28,915 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:28,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:30:28,916 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T02:30:28,920 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:28,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:30:28,925 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,927 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,935 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T02:30:28,940 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T02:30:28,946 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:30:28,947 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62787145, jitterRate=-0.0643986314535141}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T02:30:28,955 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T02:30:28,956 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T02:30:28,981 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e85d71e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:29,008 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-08T02:30:29,018 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T02:30:29,018 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T02:30:29,021 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T02:30:29,022 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T02:30:29,026 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-11-08T02:30:29,026 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T02:30:29,049 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T02:30:29,060 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T02:30:29,102 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-08T02:30:29,106 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T02:30:29,107 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T02:30:29,118 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-08T02:30:29,120 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T02:30:29,125 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T02:30:29,136 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-08T02:30:29,137 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T02:30:29,144 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T02:30:29,157 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T02:30:29,168 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T02:30:29,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T02:30:29,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T02:30:29,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:29,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:29,179 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=ecdbff65922a,40841,1731033025873, sessionid=0x10119916ccb0000, setting cluster-up flag (Was=false) 2024-11-08T02:30:29,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:29,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:29,228 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T02:30:29,230 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ecdbff65922a,40841,1731033025873 2024-11-08T02:30:29,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:29,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:29,277 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T02:30:29,280 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ecdbff65922a,40841,1731033025873 2024-11-08T02:30:29,305 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ecdbff65922a:40577 2024-11-08T02:30:29,307 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1008): ClusterId : 8df15f50-b0f4-40bc-ac8d-430988525893 2024-11-08T02:30:29,310 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T02:30:29,321 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T02:30:29,322 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T02:30:29,329 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T02:30:29,329 DEBUG [RS:0;ecdbff65922a:40577 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b807e31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:29,332 DEBUG [RS:0;ecdbff65922a:40577 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97ed05d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ecdbff65922a/172.17.0.2:0 2024-11-08T02:30:29,336 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-08T02:30:29,336 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-08T02:30:29,336 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-08T02:30:29,339 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(3073): reportForDuty to master=ecdbff65922a,40841,1731033025873 with isa=ecdbff65922a/172.17.0.2:40577, startcode=1731033026611 2024-11-08T02:30:29,353 DEBUG [RS:0;ecdbff65922a:40577 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T02:30:29,368 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-08T02:30:29,374 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-08T02:30:29,380 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T02:30:29,385 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ecdbff65922a,40841,1731033025873 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T02:30:29,389 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41411, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T02:30:29,389 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ecdbff65922a:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:30:29,389 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ecdbff65922a:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:30:29,389 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ecdbff65922a:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:30:29,389 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ecdbff65922a:0, corePoolSize=5, maxPoolSize=5 2024-11-08T02:30:29,390 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ecdbff65922a:0, corePoolSize=10, maxPoolSize=10 2024-11-08T02:30:29,390 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,390 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ecdbff65922a:0, corePoolSize=2, maxPoolSize=2 2024-11-08T02:30:29,390 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,392 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731033059392 2024-11-08T02:30:29,394 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T02:30:29,395 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T02:30:29,396 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T02:30:29,396 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40841 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3280) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:593) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:29,396 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-08T02:30:29,399 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T02:30:29,400 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T02:30:29,400 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:29,400 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T02:30:29,400 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T02:30:29,400 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T02:30:29,401 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,403 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T02:30:29,404 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T02:30:29,404 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T02:30:29,407 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T02:30:29,407 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T02:30:29,412 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ecdbff65922a:0:becomeActiveMaster-HFileCleaner.large.0-1731033029408,5,FailOnTimeoutGroup] 2024-11-08T02:30:29,413 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ecdbff65922a:0:becomeActiveMaster-HFileCleaner.small.0-1731033029413,5,FailOnTimeoutGroup] 2024-11-08T02:30:29,413 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,413 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T02:30:29,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741831_1007 (size=1039) 2024-11-08T02:30:29,415 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,415 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,423 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(3097): Master is not running yet 2024-11-08T02:30:29,424 WARN [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1099): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T02:30:29,526 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(3073): reportForDuty to master=ecdbff65922a,40841,1731033025873 with isa=ecdbff65922a/172.17.0.2:40577, startcode=1731033026611 2024-11-08T02:30:29,529 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40841 {}] master.ServerManager(332): Checking decommissioned status of RegionServer ecdbff65922a,40577,1731033026611 2024-11-08T02:30:29,533 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40841 {}] master.ServerManager(486): Registering regionserver=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:29,543 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:30:29,543 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:46319 2024-11-08T02:30:29,543 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-08T02:30:29,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T02:30:29,553 DEBUG [RS:0;ecdbff65922a:40577 {}] zookeeper.ZKUtil(111): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ecdbff65922a,40577,1731033026611 2024-11-08T02:30:29,553 WARN [RS:0;ecdbff65922a:40577 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T02:30:29,554 INFO [RS:0;ecdbff65922a:40577 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T02:30:29,554 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/WALs/ecdbff65922a,40577,1731033026611 2024-11-08T02:30:29,556 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ecdbff65922a,40577,1731033026611] 2024-11-08T02:30:29,571 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-08T02:30:29,581 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T02:30:29,595 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T02:30:29,597 INFO [RS:0;ecdbff65922a:40577 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T02:30:29,597 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,598 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-08T02:30:29,603 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,604 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,604 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,604 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,604 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,605 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,605 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ecdbff65922a:0, corePoolSize=2, maxPoolSize=2 2024-11-08T02:30:29,605 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,605 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,605 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,605 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,606 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ecdbff65922a:0, corePoolSize=1, maxPoolSize=1 2024-11-08T02:30:29,606 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ecdbff65922a:0, corePoolSize=3, maxPoolSize=3 2024-11-08T02:30:29,606 DEBUG [RS:0;ecdbff65922a:40577 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0, corePoolSize=3, maxPoolSize=3 2024-11-08T02:30:29,607 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,607 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,607 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,608 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,608 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40577,1731033026611-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T02:30:29,626 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T02:30:29,628 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40577,1731033026611-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:29,645 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.Replication(204): ecdbff65922a,40577,1731033026611 started 2024-11-08T02:30:29,645 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1767): Serving as ecdbff65922a,40577,1731033026611, RpcServer on ecdbff65922a/172.17.0.2:40577, sessionid=0x10119916ccb0001 2024-11-08T02:30:29,646 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T02:30:29,646 DEBUG [RS:0;ecdbff65922a:40577 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ecdbff65922a,40577,1731033026611 2024-11-08T02:30:29,646 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ecdbff65922a,40577,1731033026611' 2024-11-08T02:30:29,646 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T02:30:29,647 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T02:30:29,648 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T02:30:29,648 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T02:30:29,648 DEBUG [RS:0;ecdbff65922a:40577 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ecdbff65922a,40577,1731033026611 2024-11-08T02:30:29,648 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ecdbff65922a,40577,1731033026611' 2024-11-08T02:30:29,648 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T02:30:29,649 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T02:30:29,649 DEBUG [RS:0;ecdbff65922a:40577 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T02:30:29,649 INFO [RS:0;ecdbff65922a:40577 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T02:30:29,649 INFO [RS:0;ecdbff65922a:40577 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T02:30:29,756 INFO [RS:0;ecdbff65922a:40577 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T02:30:29,759 INFO [RS:0;ecdbff65922a:40577 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ecdbff65922a%2C40577%2C1731033026611, suffix=, logDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/WALs/ecdbff65922a,40577,1731033026611, archiveDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/oldWALs, maxLogs=32 2024-11-08T02:30:29,774 DEBUG [RS:0;ecdbff65922a:40577 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/WALs/ecdbff65922a,40577,1731033026611/ecdbff65922a%2C40577%2C1731033026611.1731033029761, exclude list is [], retry=0 2024-11-08T02:30:29,778 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39161,DS-486aa015-d11c-4a8a-a692-851a6f1d3552,DISK] 2024-11-08T02:30:29,781 INFO [RS:0;ecdbff65922a:40577 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/WALs/ecdbff65922a,40577,1731033026611/ecdbff65922a%2C40577%2C1731033026611.1731033029761 2024-11-08T02:30:29,781 DEBUG [RS:0;ecdbff65922a:40577 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43907:43907)] 2024-11-08T02:30:29,818 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-08T02:30:29,818 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:30:29,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741833_1009 (size=32) 2024-11-08T02:30:30,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:30,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T02:30:30,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T02:30:30,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:30,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:30:30,246 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T02:30:30,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T02:30:30,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:30,250 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:30:30,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T02:30:30,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T02:30:30,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:30,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:30:30,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740 2024-11-08T02:30:30,259 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740 2024-11-08T02:30:30,263 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:30:30,266 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T02:30:30,270 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:30:30,272 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62147075, jitterRate=-0.07393641769886017}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:30:30,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T02:30:30,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T02:30:30,275 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T02:30:30,275 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T02:30:30,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T02:30:30,276 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T02:30:30,277 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T02:30:30,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T02:30:30,280 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-08T02:30:30,280 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-08T02:30:30,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T02:30:30,293 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T02:30:30,295 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T02:30:30,449 DEBUG [ecdbff65922a:40841 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T02:30:30,456 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:30,461 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ecdbff65922a,40577,1731033026611, state=OPENING 2024-11-08T02:30:30,502 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T02:30:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:30,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:30:30,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:30:30,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:30:30,696 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:30,698 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T02:30:30,701 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42806, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T02:30:30,712 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-08T02:30:30,712 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T02:30:30,713 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T02:30:30,716 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ecdbff65922a%2C40577%2C1731033026611.meta, suffix=.meta, logDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/WALs/ecdbff65922a,40577,1731033026611, archiveDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/oldWALs, maxLogs=32 2024-11-08T02:30:30,732 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/WALs/ecdbff65922a,40577,1731033026611/ecdbff65922a%2C40577%2C1731033026611.meta.1731033030718.meta, exclude list is [], retry=0 2024-11-08T02:30:30,737 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39161,DS-486aa015-d11c-4a8a-a692-851a6f1d3552,DISK] 2024-11-08T02:30:30,740 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/WALs/ecdbff65922a,40577,1731033026611/ecdbff65922a%2C40577%2C1731033026611.meta.1731033030718.meta 2024-11-08T02:30:30,740 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43907:43907)] 2024-11-08T02:30:30,741 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:30:30,742 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T02:30:30,795 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T02:30:30,799 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T02:30:30,802 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T02:30:30,802 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:30,803 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-08T02:30:30,803 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-08T02:30:30,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T02:30:30,808 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T02:30:30,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:30,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:30:30,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T02:30:30,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T02:30:30,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:30,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:30:30,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T02:30:30,814 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T02:30:30,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:30,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T02:30:30,817 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740 2024-11-08T02:30:30,820 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740 2024-11-08T02:30:30,823 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:30:30,826 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-08T02:30:30,827 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73456174, jitterRate=0.09458228945732117}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:30:30,829 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-08T02:30:30,835 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731033030692 2024-11-08T02:30:30,845 DEBUG [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T02:30:30,845 INFO [RS_OPEN_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-08T02:30:30,846 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:30,848 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ecdbff65922a,40577,1731033026611, state=OPEN 2024-11-08T02:30:30,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T02:30:30,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T02:30:30,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:30:30,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T02:30:30,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T02:30:30,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=ecdbff65922a,40577,1731033026611 in 363 msec 2024-11-08T02:30:30,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T02:30:30,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 601 msec 2024-11-08T02:30:30,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.5760 sec 2024-11-08T02:30:30,898 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731033030898, completionTime=-1 2024-11-08T02:30:30,899 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T02:30:30,899 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-08T02:30:30,929 DEBUG [hconnection-0x616dbb2c-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:30,932 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42808, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:30,941 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-08T02:30:30,941 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731033090941 2024-11-08T02:30:30,941 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731033150941 2024-11-08T02:30:30,941 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 42 msec 2024-11-08T02:30:30,979 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40841,1731033025873-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:30,979 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40841,1731033025873-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:30,979 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40841,1731033025873-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:30,981 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ecdbff65922a:40841, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:30,981 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:30,986 DEBUG [master/ecdbff65922a:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-08T02:30:30,989 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-08T02:30:30,990 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T02:30:30,996 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-08T02:30:30,999 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:30:31,000 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:31,002 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:30:31,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741835_1011 (size=358) 2024-11-08T02:30:31,422 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 934a9f648376cf09654e404de226250e, NAME => 'hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:30:31,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741836_1012 (size=42) 2024-11-08T02:30:31,835 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:31,836 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 934a9f648376cf09654e404de226250e, disabling compactions & flushes 2024-11-08T02:30:31,836 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:30:31,836 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:30:31,836 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. after waiting 0 ms 2024-11-08T02:30:31,836 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:30:31,836 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:30:31,837 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 934a9f648376cf09654e404de226250e: 2024-11-08T02:30:31,841 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:30:31,850 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731033031842"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731033031842"}]},"ts":"1731033031842"} 2024-11-08T02:30:31,869 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T02:30:31,870 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:30:31,873 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033031871"}]},"ts":"1731033031871"} 2024-11-08T02:30:31,877 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-08T02:30:31,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=934a9f648376cf09654e404de226250e, ASSIGN}] 2024-11-08T02:30:31,909 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=934a9f648376cf09654e404de226250e, ASSIGN 2024-11-08T02:30:31,911 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=934a9f648376cf09654e404de226250e, ASSIGN; state=OFFLINE, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=false 2024-11-08T02:30:32,063 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=934a9f648376cf09654e404de226250e, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:32,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 934a9f648376cf09654e404de226250e, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:30:32,229 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:32,238 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:30:32,238 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 934a9f648376cf09654e404de226250e, NAME => 'hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:30:32,239 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 934a9f648376cf09654e404de226250e 2024-11-08T02:30:32,239 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:32,240 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 934a9f648376cf09654e404de226250e 2024-11-08T02:30:32,240 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 934a9f648376cf09654e404de226250e 2024-11-08T02:30:32,243 INFO [StoreOpener-934a9f648376cf09654e404de226250e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 934a9f648376cf09654e404de226250e 2024-11-08T02:30:32,245 INFO [StoreOpener-934a9f648376cf09654e404de226250e-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 934a9f648376cf09654e404de226250e columnFamilyName info 2024-11-08T02:30:32,245 DEBUG [StoreOpener-934a9f648376cf09654e404de226250e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:32,246 INFO [StoreOpener-934a9f648376cf09654e404de226250e-1 {}] regionserver.HStore(327): Store=934a9f648376cf09654e404de226250e/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:30:32,248 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e 2024-11-08T02:30:32,249 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e 2024-11-08T02:30:32,253 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 934a9f648376cf09654e404de226250e 2024-11-08T02:30:32,257 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:30:32,259 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 934a9f648376cf09654e404de226250e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59000141, jitterRate=-0.12082938849925995}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T02:30:32,260 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 934a9f648376cf09654e404de226250e: 2024-11-08T02:30:32,263 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e., pid=6, masterSystemTime=1731033032228 2024-11-08T02:30:32,266 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:30:32,266 INFO [RS_OPEN_PRIORITY_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:30:32,267 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=934a9f648376cf09654e404de226250e, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:32,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T02:30:32,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 934a9f648376cf09654e404de226250e, server=ecdbff65922a,40577,1731033026611 in 198 msec 2024-11-08T02:30:32,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T02:30:32,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=934a9f648376cf09654e404de226250e, ASSIGN in 371 msec 2024-11-08T02:30:32,281 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:30:32,281 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033032281"}]},"ts":"1731033032281"} 2024-11-08T02:30:32,284 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-08T02:30:32,338 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-08T02:30:32,340 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:30:32,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:32,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-08T02:30:32,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:30:32,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3490 sec 2024-11-08T02:30:32,370 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-08T02:30:32,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T02:30:32,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 40 msec 2024-11-08T02:30:32,414 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-08T02:30:32,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-08T02:30:32,450 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 33 msec 2024-11-08T02:30:32,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-08T02:30:32,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-08T02:30:32,495 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 5.649sec 2024-11-08T02:30:32,497 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T02:30:32,500 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T02:30:32,503 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T02:30:32,504 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T02:30:32,504 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T02:30:32,507 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40841,1731033025873-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T02:30:32,508 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40841,1731033025873-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T02:30:32,515 DEBUG [master/ecdbff65922a:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-08T02:30:32,516 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T02:30:32,516 INFO [master/ecdbff65922a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ecdbff65922a,40841,1731033025873-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T02:30:32,611 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e541e88 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e83c466 2024-11-08T02:30:32,611 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-08T02:30:32,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305a704f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:32,624 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T02:30:32,624 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T02:30:32,633 DEBUG [hconnection-0x68773b0e-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:32,641 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42812, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:32,649 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=ecdbff65922a,40841,1731033025873 2024-11-08T02:30:32,662 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMixedAtomicity Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=117, ProcessCount=11, AvailableMemoryMB=2864 2024-11-08T02:30:32,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:30:32,675 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51318, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:30:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:30:32,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:30:32,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:30:32,690 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:30:32,690 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:32,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 9 2024-11-08T02:30:32,693 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:30:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T02:30:32,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741837_1013 (size=960) 2024-11-08T02:30:32,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T02:30:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T02:30:33,139 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:30:33,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741838_1014 (size=53) 2024-11-08T02:30:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T02:30:33,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:33,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing dc8ad63d9a6a95258392be1a735a16e9, disabling compactions & flushes 2024-11-08T02:30:33,555 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,555 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. after waiting 0 ms 2024-11-08T02:30:33,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,556 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,556 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:33,559 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:30:33,560 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731033033560"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731033033560"}]},"ts":"1731033033560"} 2024-11-08T02:30:33,564 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T02:30:33,567 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:30:33,567 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033033567"}]},"ts":"1731033033567"} 2024-11-08T02:30:33,569 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:30:33,619 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc8ad63d9a6a95258392be1a735a16e9, ASSIGN}] 2024-11-08T02:30:33,622 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc8ad63d9a6a95258392be1a735a16e9, ASSIGN 2024-11-08T02:30:33,624 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc8ad63d9a6a95258392be1a735a16e9, ASSIGN; state=OFFLINE, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=false 2024-11-08T02:30:33,775 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=dc8ad63d9a6a95258392be1a735a16e9, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:33,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:30:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T02:30:33,933 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:33,942 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,942 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:30:33,942 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,943 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:30:33,943 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,943 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,945 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,948 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:30:33,949 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc8ad63d9a6a95258392be1a735a16e9 columnFamilyName A 2024-11-08T02:30:33,949 DEBUG [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:33,950 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.HStore(327): Store=dc8ad63d9a6a95258392be1a735a16e9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:30:33,950 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,952 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:30:33,953 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc8ad63d9a6a95258392be1a735a16e9 columnFamilyName B 2024-11-08T02:30:33,953 DEBUG [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:33,953 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.HStore(327): Store=dc8ad63d9a6a95258392be1a735a16e9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:30:33,954 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,955 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:30:33,956 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc8ad63d9a6a95258392be1a735a16e9 columnFamilyName C 2024-11-08T02:30:33,956 DEBUG [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:30:33,957 INFO [StoreOpener-dc8ad63d9a6a95258392be1a735a16e9-1 {}] regionserver.HStore(327): Store=dc8ad63d9a6a95258392be1a735a16e9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:30:33,957 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,958 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,959 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,961 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:30:33,963 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:33,967 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:30:33,968 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened dc8ad63d9a6a95258392be1a735a16e9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59073816, jitterRate=-0.11973154544830322}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:30:33,968 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:33,970 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., pid=11, masterSystemTime=1731033033932 2024-11-08T02:30:33,972 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,972 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:33,973 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=dc8ad63d9a6a95258392be1a735a16e9, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:33,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T02:30:33,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 in 200 msec 2024-11-08T02:30:33,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T02:30:33,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc8ad63d9a6a95258392be1a735a16e9, ASSIGN in 363 msec 2024-11-08T02:30:33,986 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:30:33,987 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033033987"}]},"ts":"1731033033987"} 2024-11-08T02:30:33,989 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:30:33,995 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:30:33,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.3090 sec 2024-11-08T02:30:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-08T02:30:34,848 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 9 completed 2024-11-08T02:30:34,856 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f6e36fe to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e98ea32 2024-11-08T02:30:34,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b9fcedf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:34,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:34,902 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42828, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:34,907 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:30:34,911 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51330, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:30:34,922 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f343a4d to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12885408 2024-11-08T02:30:34,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9bd0964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:34,930 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22cb07dd to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72b32f98 2024-11-08T02:30:34,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18cb251d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:34,939 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x478bae6b to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4977266 2024-11-08T02:30:34,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b55c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:34,955 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5400112e to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bbb5d8a 2024-11-08T02:30:34,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e52b42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:34,974 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38766d64 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18603bb9 2024-11-08T02:30:34,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3883f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:34,989 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x295cb1ac to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72e97e4b 2024-11-08T02:30:35,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a1285d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:35,003 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70267494 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@490457fd 2024-11-08T02:30:35,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@527c6d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:35,014 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d2a8e08 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c8de680 2024-11-08T02:30:35,027 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fe2fa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:35,030 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c915d17 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f6b07e3 2024-11-08T02:30:35,044 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595e9ebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:30:35,050 DEBUG [hconnection-0x64e9f767-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,051 DEBUG [hconnection-0x6e2d0668-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,051 DEBUG [hconnection-0x71465415-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,054 DEBUG [hconnection-0x5331b0f7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,054 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42844, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,055 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42830, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,055 DEBUG [hconnection-0x210b4d19-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,055 DEBUG [hconnection-0x7782676d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,056 DEBUG [hconnection-0x61efc5aa-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,056 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42852, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,056 DEBUG [hconnection-0x6aaf36c5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,057 DEBUG [hconnection-0xc43284f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:30:35,058 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42860, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,058 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42866, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,059 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42880, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:35,062 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42884, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees 2024-11-08T02:30:35,068 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42892, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,069 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42894, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:30:35,069 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:35,072 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T02:30:35,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:35,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:35,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:30:35,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:35,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:35,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:35,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:35,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:35,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T02:30:35,234 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-08T02:30:35,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:35,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:35,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:35,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8e51aa4859354965ae3aa4ea16fe5d6a is 50, key is test_row_0/A:col10/1731033035131/Put/seqid=0 2024-11-08T02:30:35,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033095296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033095296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033095300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033095304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033095304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741839_1015 (size=12001) 2024-11-08T02:30:35,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8e51aa4859354965ae3aa4ea16fe5d6a 2024-11-08T02:30:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T02:30:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033095418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,421 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033095418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033095418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-08T02:30:35,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033095420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033095419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:35,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:35,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/087a079dc85448dfa456ce6acefc97f1 is 50, key is test_row_0/B:col10/1731033035131/Put/seqid=0 2024-11-08T02:30:35,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741840_1016 (size=12001) 2024-11-08T02:30:35,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/087a079dc85448dfa456ce6acefc97f1 2024-11-08T02:30:35,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/7facc17547e9424eb7e99849ed8e6081 is 50, key is test_row_0/C:col10/1731033035131/Put/seqid=0 2024-11-08T02:30:35,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741841_1017 (size=12001) 2024-11-08T02:30:35,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/7facc17547e9424eb7e99849ed8e6081 2024-11-08T02:30:35,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8e51aa4859354965ae3aa4ea16fe5d6a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e51aa4859354965ae3aa4ea16fe5d6a 2024-11-08T02:30:35,577 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T02:30:35,579 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T02:30:35,579 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-08T02:30:35,583 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-08T02:30:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:35,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:35,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e51aa4859354965ae3aa4ea16fe5d6a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:30:35,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:35,597 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1000): StoreScanner already closing. There is no need to updateReaders 2024-11-08T02:30:35,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/087a079dc85448dfa456ce6acefc97f1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/087a079dc85448dfa456ce6acefc97f1 2024-11-08T02:30:35,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/087a079dc85448dfa456ce6acefc97f1, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:30:35,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/7facc17547e9424eb7e99849ed8e6081 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/7facc17547e9424eb7e99849ed8e6081 2024-11-08T02:30:35,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033095626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033095626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033095627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033095628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033095629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/7facc17547e9424eb7e99849ed8e6081, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T02:30:35,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dc8ad63d9a6a95258392be1a735a16e9 in 507ms, sequenceid=13, compaction requested=false 2024-11-08T02:30:35,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T02:30:35,745 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-08T02:30:35,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:35,747 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:30:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:35,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:35,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:35,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:35,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/bf91c28a7e7643ad843808d15b26d8b9 is 50, key is test_row_0/A:col10/1731033035299/Put/seqid=0 2024-11-08T02:30:35,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741842_1018 (size=12001) 2024-11-08T02:30:35,774 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/bf91c28a7e7643ad843808d15b26d8b9 2024-11-08T02:30:35,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/c5c9a58bec5a4ef0801ee4eb89e12dd7 is 50, key is test_row_0/B:col10/1731033035299/Put/seqid=0 2024-11-08T02:30:35,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741843_1019 (size=12001) 2024-11-08T02:30:35,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:35,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:35,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033095973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033095976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033095978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033095981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:35,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033095983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033096089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033096089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033096091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033096091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033096091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T02:30:36,238 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/c5c9a58bec5a4ef0801ee4eb89e12dd7 2024-11-08T02:30:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/3e7a9d374e79465bb013ea80a6159048 is 50, key is test_row_0/C:col10/1731033035299/Put/seqid=0 2024-11-08T02:30:36,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741844_1020 (size=12001) 2024-11-08T02:30:36,300 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/3e7a9d374e79465bb013ea80a6159048 2024-11-08T02:30:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033096298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033096299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033096300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033096301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033096302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/bf91c28a7e7643ad843808d15b26d8b9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bf91c28a7e7643ad843808d15b26d8b9 2024-11-08T02:30:36,315 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:30:36,330 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bf91c28a7e7643ad843808d15b26d8b9, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T02:30:36,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/c5c9a58bec5a4ef0801ee4eb89e12dd7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c5c9a58bec5a4ef0801ee4eb89e12dd7 2024-11-08T02:30:36,357 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c5c9a58bec5a4ef0801ee4eb89e12dd7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T02:30:36,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/3e7a9d374e79465bb013ea80a6159048 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3e7a9d374e79465bb013ea80a6159048 2024-11-08T02:30:36,381 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3e7a9d374e79465bb013ea80a6159048, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T02:30:36,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dc8ad63d9a6a95258392be1a735a16e9 in 637ms, sequenceid=37, compaction requested=false 2024-11-08T02:30:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13 2024-11-08T02:30:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=13 2024-11-08T02:30:36,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-08T02:30:36,393 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3150 sec 2024-11-08T02:30:36,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees in 1.3320 sec 2024-11-08T02:30:36,417 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T02:30:36,418 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T02:30:36,420 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-08T02:30:36,420 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-08T02:30:36,421 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T02:30:36,421 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T02:30:36,422 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T02:30:36,422 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T02:30:36,423 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T02:30:36,423 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T02:30:36,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:36,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:30:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:36,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8ab7ec653ba24612a43f2cc86f5429e3 is 50, key is test_row_0/A:col10/1731033035968/Put/seqid=0 2024-11-08T02:30:36,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741845_1021 (size=12001) 2024-11-08T02:30:36,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8ab7ec653ba24612a43f2cc86f5429e3 2024-11-08T02:30:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033096665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033096665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033096667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033096673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033096674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/d4c3641e887e4116805ce2e601d66f12 is 50, key is test_row_0/B:col10/1731033035968/Put/seqid=0 2024-11-08T02:30:36,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741846_1022 (size=12001) 2024-11-08T02:30:36,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/d4c3641e887e4116805ce2e601d66f12 2024-11-08T02:30:36,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/e028be4d907048afadf39b845aabcc09 is 50, key is test_row_0/C:col10/1731033035968/Put/seqid=0 2024-11-08T02:30:36,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741847_1023 (size=12001) 2024-11-08T02:30:36,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/e028be4d907048afadf39b845aabcc09 2024-11-08T02:30:36,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033096776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033096778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033096779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033096787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8ab7ec653ba24612a43f2cc86f5429e3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8ab7ec653ba24612a43f2cc86f5429e3 2024-11-08T02:30:36,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033096789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:36,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8ab7ec653ba24612a43f2cc86f5429e3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T02:30:36,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/d4c3641e887e4116805ce2e601d66f12 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d4c3641e887e4116805ce2e601d66f12 2024-11-08T02:30:36,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d4c3641e887e4116805ce2e601d66f12, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T02:30:36,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/e028be4d907048afadf39b845aabcc09 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e028be4d907048afadf39b845aabcc09 2024-11-08T02:30:36,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e028be4d907048afadf39b845aabcc09, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T02:30:36,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for dc8ad63d9a6a95258392be1a735a16e9 in 236ms, sequenceid=51, compaction requested=true 2024-11-08T02:30:36,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:36,851 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:36,852 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:36,855 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:36,857 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:36,857 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:36,857 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/087a079dc85448dfa456ce6acefc97f1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c5c9a58bec5a4ef0801ee4eb89e12dd7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d4c3641e887e4116805ce2e601d66f12] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=35.2 K 2024-11-08T02:30:36,859 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 087a079dc85448dfa456ce6acefc97f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731033035120 2024-11-08T02:30:36,860 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c5c9a58bec5a4ef0801ee4eb89e12dd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731033035295 2024-11-08T02:30:36,862 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:36,862 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting d4c3641e887e4116805ce2e601d66f12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731033035968 2024-11-08T02:30:36,863 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:36,863 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:36,863 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e51aa4859354965ae3aa4ea16fe5d6a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bf91c28a7e7643ad843808d15b26d8b9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8ab7ec653ba24612a43f2cc86f5429e3] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=35.2 K 2024-11-08T02:30:36,865 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8e51aa4859354965ae3aa4ea16fe5d6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731033035120 2024-11-08T02:30:36,866 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting bf91c28a7e7643ad843808d15b26d8b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731033035295 2024-11-08T02:30:36,875 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8ab7ec653ba24612a43f2cc86f5429e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731033035968 2024-11-08T02:30:36,904 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#9 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:36,905 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:36,905 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/e87cf96fc525412dac1435f5dad1b2eb is 50, key is test_row_0/A:col10/1731033035968/Put/seqid=0 2024-11-08T02:30:36,906 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/90375c3fa5e04beab6ee277c69afde4f is 50, key is test_row_0/B:col10/1731033035968/Put/seqid=0 2024-11-08T02:30:36,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741849_1025 (size=12104) 2024-11-08T02:30:36,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741848_1024 (size=12104) 2024-11-08T02:30:36,938 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/e87cf96fc525412dac1435f5dad1b2eb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e87cf96fc525412dac1435f5dad1b2eb 2024-11-08T02:30:36,965 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into e87cf96fc525412dac1435f5dad1b2eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:36,965 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:36,965 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033036850; duration=0sec 2024-11-08T02:30:36,966 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:36,966 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:36,966 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:36,969 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:36,970 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:36,970 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:36,970 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/7facc17547e9424eb7e99849ed8e6081, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3e7a9d374e79465bb013ea80a6159048, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e028be4d907048afadf39b845aabcc09] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=35.2 K 2024-11-08T02:30:36,971 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7facc17547e9424eb7e99849ed8e6081, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731033035120 2024-11-08T02:30:36,972 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3e7a9d374e79465bb013ea80a6159048, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731033035295 2024-11-08T02:30:36,976 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting e028be4d907048afadf39b845aabcc09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731033035968 2024-11-08T02:30:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:36,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:30:36,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:36,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:36,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:36,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:36,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:36,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:37,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/bb05790f2f1043a5b07a7d7a7a276d67 is 50, key is test_row_0/A:col10/1731033036660/Put/seqid=0 2024-11-08T02:30:37,009 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#12 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:37,010 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/c3622dab435b401fbe3b6f3e8d9f37d0 is 50, key is test_row_0/C:col10/1731033035968/Put/seqid=0 2024-11-08T02:30:37,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033097008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033097009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033097011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033097012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033097015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741850_1026 (size=12104) 2024-11-08T02:30:37,050 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/c3622dab435b401fbe3b6f3e8d9f37d0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c3622dab435b401fbe3b6f3e8d9f37d0 2024-11-08T02:30:37,062 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into c3622dab435b401fbe3b6f3e8d9f37d0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:37,062 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:37,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741851_1027 (size=12001) 2024-11-08T02:30:37,063 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033036851; duration=0sec 2024-11-08T02:30:37,064 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:37,064 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:37,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033097117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033097118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033097122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033097123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033097124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-08T02:30:37,189 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 12 completed 2024-11-08T02:30:37,192 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:37,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees 2024-11-08T02:30:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-08T02:30:37,197 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:37,198 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:37,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:37,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-08T02:30:37,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033097322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033097324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033097329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033097330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033097329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,346 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/90375c3fa5e04beab6ee277c69afde4f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/90375c3fa5e04beab6ee277c69afde4f 2024-11-08T02:30:37,353 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-11-08T02:30:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:37,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,365 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into 90375c3fa5e04beab6ee277c69afde4f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:37,365 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:37,365 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033036851; duration=0sec 2024-11-08T02:30:37,366 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:37,367 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:37,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/bb05790f2f1043a5b07a7d7a7a276d67 2024-11-08T02:30:37,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/55f40b39e06149d5a8597881d3bb5e15 is 50, key is test_row_0/B:col10/1731033036660/Put/seqid=0 2024-11-08T02:30:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-08T02:30:37,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741852_1028 (size=12001) 2024-11-08T02:30:37,508 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-11-08T02:30:37,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033097626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033097630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033097636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033097637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033097637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,663 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-11-08T02:30:37,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-08T02:30:37,819 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-11-08T02:30:37,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:37,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/55f40b39e06149d5a8597881d3bb5e15 2024-11-08T02:30:37,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/945bc43e0fc942718e5a479fd9f533ae is 50, key is test_row_0/C:col10/1731033036660/Put/seqid=0 2024-11-08T02:30:37,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741853_1029 (size=12001) 2024-11-08T02:30:37,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/945bc43e0fc942718e5a479fd9f533ae 2024-11-08T02:30:37,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/bb05790f2f1043a5b07a7d7a7a276d67 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bb05790f2f1043a5b07a7d7a7a276d67 2024-11-08T02:30:37,974 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:37,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-11-08T02:30:37,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:37,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:37,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] handler.RSProcedureHandler(58): pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=15 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=15 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:37,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bb05790f2f1043a5b07a7d7a7a276d67, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:30:37,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/55f40b39e06149d5a8597881d3bb5e15 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55f40b39e06149d5a8597881d3bb5e15 2024-11-08T02:30:37,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55f40b39e06149d5a8597881d3bb5e15, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:30:37,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/945bc43e0fc942718e5a479fd9f533ae as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/945bc43e0fc942718e5a479fd9f533ae 2024-11-08T02:30:38,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/945bc43e0fc942718e5a479fd9f533ae, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:30:38,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dc8ad63d9a6a95258392be1a735a16e9 in 1029ms, sequenceid=76, compaction requested=false 2024-11-08T02:30:38,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:38,130 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-11-08T02:30:38,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:38,133 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:30:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:38,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:38,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:38,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/0f64faca361c42e18ffca4dee99a7da5 is 50, key is test_row_0/A:col10/1731033037006/Put/seqid=0 2024-11-08T02:30:38,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741854_1030 (size=12001) 2024-11-08T02:30:38,168 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/0f64faca361c42e18ffca4dee99a7da5 2024-11-08T02:30:38,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/74bc7a9971d740a4a293936f19fb5ffe is 50, key is test_row_0/B:col10/1731033037006/Put/seqid=0 2024-11-08T02:30:38,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033098183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033098180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033098188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033098190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033098193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741855_1031 (size=12001) 2024-11-08T02:30:38,204 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/74bc7a9971d740a4a293936f19fb5ffe 2024-11-08T02:30:38,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/d53fc06a86694b1d915e0fd8bf3e87c9 is 50, key is test_row_0/C:col10/1731033037006/Put/seqid=0 2024-11-08T02:30:38,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741856_1032 (size=12001) 2024-11-08T02:30:38,258 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/d53fc06a86694b1d915e0fd8bf3e87c9 2024-11-08T02:30:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/0f64faca361c42e18ffca4dee99a7da5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0f64faca361c42e18ffca4dee99a7da5 2024-11-08T02:30:38,287 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0f64faca361c42e18ffca4dee99a7da5, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T02:30:38,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/74bc7a9971d740a4a293936f19fb5ffe as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/74bc7a9971d740a4a293936f19fb5ffe 2024-11-08T02:30:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033098296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-08T02:30:38,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033098297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033098297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033098297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033098299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,307 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/74bc7a9971d740a4a293936f19fb5ffe, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T02:30:38,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/d53fc06a86694b1d915e0fd8bf3e87c9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/d53fc06a86694b1d915e0fd8bf3e87c9 2024-11-08T02:30:38,326 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/d53fc06a86694b1d915e0fd8bf3e87c9, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T02:30:38,329 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dc8ad63d9a6a95258392be1a735a16e9 in 196ms, sequenceid=90, compaction requested=true 2024-11-08T02:30:38,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:38,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:38,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-08T02:30:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=15 2024-11-08T02:30:38,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14 2024-11-08T02:30:38,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1320 sec 2024-11-08T02:30:38,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees in 1.1450 sec 2024-11-08T02:30:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:38,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:30:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:38,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/e572085a331e4ccab0498acb1328af5a is 50, key is test_row_0/A:col10/1731033038176/Put/seqid=0 2024-11-08T02:30:38,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033098519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033098519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033098520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033098522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033098530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741857_1033 (size=12001) 2024-11-08T02:30:38,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/e572085a331e4ccab0498acb1328af5a 2024-11-08T02:30:38,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/f14de3ee0d3e4505af15679444fdec89 is 50, key is test_row_0/B:col10/1731033038176/Put/seqid=0 2024-11-08T02:30:38,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741858_1034 (size=12001) 2024-11-08T02:30:38,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/f14de3ee0d3e4505af15679444fdec89 2024-11-08T02:30:38,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6078050be37d4e6cb00883789f09a864 is 50, key is test_row_0/C:col10/1731033038176/Put/seqid=0 2024-11-08T02:30:38,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741859_1035 (size=12001) 2024-11-08T02:30:38,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033098629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033098630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033098631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033098635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033098636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033098835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033098836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033098839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033098839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:38,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:38,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033098840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6078050be37d4e6cb00883789f09a864 2024-11-08T02:30:39,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/e572085a331e4ccab0498acb1328af5a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e572085a331e4ccab0498acb1328af5a 2024-11-08T02:30:39,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e572085a331e4ccab0498acb1328af5a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:30:39,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/f14de3ee0d3e4505af15679444fdec89 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f14de3ee0d3e4505af15679444fdec89 2024-11-08T02:30:39,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f14de3ee0d3e4505af15679444fdec89, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:30:39,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6078050be37d4e6cb00883789f09a864 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6078050be37d4e6cb00883789f09a864 2024-11-08T02:30:39,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6078050be37d4e6cb00883789f09a864, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:30:39,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dc8ad63d9a6a95258392be1a735a16e9 in 588ms, sequenceid=115, compaction requested=true 2024-11-08T02:30:39,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:39,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:39,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:39,093 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:30:39,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:39,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:39,093 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:30:39,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:39,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:39,097 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:30:39,097 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:39,097 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,097 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/90375c3fa5e04beab6ee277c69afde4f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55f40b39e06149d5a8597881d3bb5e15, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/74bc7a9971d740a4a293936f19fb5ffe, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f14de3ee0d3e4505af15679444fdec89] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=47.0 K 2024-11-08T02:30:39,098 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:30:39,098 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:39,098 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,099 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e87cf96fc525412dac1435f5dad1b2eb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bb05790f2f1043a5b07a7d7a7a276d67, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0f64faca361c42e18ffca4dee99a7da5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e572085a331e4ccab0498acb1328af5a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=47.0 K 2024-11-08T02:30:39,099 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 90375c3fa5e04beab6ee277c69afde4f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731033035968 2024-11-08T02:30:39,100 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting e87cf96fc525412dac1435f5dad1b2eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731033035968 2024-11-08T02:30:39,100 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting bb05790f2f1043a5b07a7d7a7a276d67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033036660 2024-11-08T02:30:39,100 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 55f40b39e06149d5a8597881d3bb5e15, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033036660 2024-11-08T02:30:39,101 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0f64faca361c42e18ffca4dee99a7da5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731033036990 2024-11-08T02:30:39,101 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 74bc7a9971d740a4a293936f19fb5ffe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731033036990 2024-11-08T02:30:39,102 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting e572085a331e4ccab0498acb1328af5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033038176 2024-11-08T02:30:39,102 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f14de3ee0d3e4505af15679444fdec89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033038176 2024-11-08T02:30:39,130 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:39,131 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/7c63a5dbdf1f4b32b26bd1faafbaf9fe is 50, key is test_row_0/B:col10/1731033038176/Put/seqid=0 2024-11-08T02:30:39,132 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:39,133 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/f784536f74524ae8a1b418a70a890dc2 is 50, key is test_row_0/A:col10/1731033038176/Put/seqid=0 2024-11-08T02:30:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:39,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:30:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:39,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:39,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:39,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741860_1036 (size=12241) 2024-11-08T02:30:39,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741861_1037 (size=12241) 2024-11-08T02:30:39,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/1e132f0f974c4f9590fe439a676cb93d is 50, key is test_row_0/A:col10/1731033038519/Put/seqid=0 2024-11-08T02:30:39,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741862_1038 (size=12001) 2024-11-08T02:30:39,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033099242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033099244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033099248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033099248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033099248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-08T02:30:39,303 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 14 completed 2024-11-08T02:30:39,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees 2024-11-08T02:30:39,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-08T02:30:39,311 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:39,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:39,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033099350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033099350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033099353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033099354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033099356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-08T02:30:39,464 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-08T02:30:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(58): pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033099555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033099557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033099557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033099555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033099559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,577 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/f784536f74524ae8a1b418a70a890dc2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f784536f74524ae8a1b418a70a890dc2 2024-11-08T02:30:39,590 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into f784536f74524ae8a1b418a70a890dc2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:39,590 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:39,590 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=12, startTime=1731033039093; duration=0sec 2024-11-08T02:30:39,590 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:39,590 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:39,590 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:30:39,594 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/7c63a5dbdf1f4b32b26bd1faafbaf9fe as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7c63a5dbdf1f4b32b26bd1faafbaf9fe 2024-11-08T02:30:39,596 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:30:39,596 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:39,596 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,596 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c3622dab435b401fbe3b6f3e8d9f37d0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/945bc43e0fc942718e5a479fd9f533ae, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/d53fc06a86694b1d915e0fd8bf3e87c9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6078050be37d4e6cb00883789f09a864] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=47.0 K 2024-11-08T02:30:39,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/1e132f0f974c4f9590fe439a676cb93d 2024-11-08T02:30:39,599 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting c3622dab435b401fbe3b6f3e8d9f37d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731033035968 2024-11-08T02:30:39,601 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 945bc43e0fc942718e5a479fd9f533ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033036660 2024-11-08T02:30:39,603 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d53fc06a86694b1d915e0fd8bf3e87c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731033036990 2024-11-08T02:30:39,606 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6078050be37d4e6cb00883789f09a864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033038176 2024-11-08T02:30:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-08T02:30:39,613 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into 7c63a5dbdf1f4b32b26bd1faafbaf9fe(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:39,613 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:39,613 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=12, startTime=1731033039093; duration=0sec 2024-11-08T02:30:39,614 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:39,614 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:39,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/060a7fb2ad2346b1bb7e0bac7750df24 is 50, key is test_row_0/B:col10/1731033038519/Put/seqid=0 2024-11-08T02:30:39,617 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-08T02:30:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(58): pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,652 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:39,653 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/1c544aaeefcc4d94b658030f624691fb is 50, key is test_row_0/C:col10/1731033038176/Put/seqid=0 2024-11-08T02:30:39,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741863_1039 (size=12001) 2024-11-08T02:30:39,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/060a7fb2ad2346b1bb7e0bac7750df24 2024-11-08T02:30:39,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741864_1040 (size=12241) 2024-11-08T02:30:39,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/955778f4ac624705970d028afa4b2815 is 50, key is test_row_0/C:col10/1731033038519/Put/seqid=0 2024-11-08T02:30:39,686 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/1c544aaeefcc4d94b658030f624691fb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1c544aaeefcc4d94b658030f624691fb 2024-11-08T02:30:39,698 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 1c544aaeefcc4d94b658030f624691fb(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:39,698 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:39,698 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=12, startTime=1731033039093; duration=0sec 2024-11-08T02:30:39,699 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:39,699 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:39,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741865_1041 (size=12001) 2024-11-08T02:30:39,771 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-08T02:30:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(58): pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033099861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033099862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033099863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033099865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033099863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-08T02:30:39,925 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:39,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-08T02:30:39,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:39,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:39,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(58): pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:40,080 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-08T02:30:40,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:40,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(58): pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:40,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/955778f4ac624705970d028afa4b2815 2024-11-08T02:30:40,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/1e132f0f974c4f9590fe439a676cb93d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/1e132f0f974c4f9590fe439a676cb93d 2024-11-08T02:30:40,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/1e132f0f974c4f9590fe439a676cb93d, entries=150, sequenceid=128, filesize=11.7 K 2024-11-08T02:30:40,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/060a7fb2ad2346b1bb7e0bac7750df24 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/060a7fb2ad2346b1bb7e0bac7750df24 2024-11-08T02:30:40,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/060a7fb2ad2346b1bb7e0bac7750df24, entries=150, sequenceid=128, filesize=11.7 K 2024-11-08T02:30:40,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/955778f4ac624705970d028afa4b2815 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/955778f4ac624705970d028afa4b2815 2024-11-08T02:30:40,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/955778f4ac624705970d028afa4b2815, entries=150, sequenceid=128, filesize=11.7 K 2024-11-08T02:30:40,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for dc8ad63d9a6a95258392be1a735a16e9 in 1008ms, sequenceid=128, compaction requested=false 2024-11-08T02:30:40,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:40,234 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-08T02:30:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:40,236 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:30:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:40,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:40,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/69e688857a1441cfb9d62bb0aae5e2ee is 50, key is test_row_0/A:col10/1731033039234/Put/seqid=0 2024-11-08T02:30:40,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741866_1042 (size=12151) 2024-11-08T02:30:40,281 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/69e688857a1441cfb9d62bb0aae5e2ee 2024-11-08T02:30:40,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fea1c374518d472fa6470e99d9518d83 is 50, key is test_row_0/B:col10/1731033039234/Put/seqid=0 2024-11-08T02:30:40,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741867_1043 (size=12151) 2024-11-08T02:30:40,315 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fea1c374518d472fa6470e99d9518d83 2024-11-08T02:30:40,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/9d2dbb80f70a463ca0c35bb79a531059 is 50, key is test_row_0/C:col10/1731033039234/Put/seqid=0 2024-11-08T02:30:40,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741868_1044 (size=12151) 2024-11-08T02:30:40,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/9d2dbb80f70a463ca0c35bb79a531059 2024-11-08T02:30:40,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/69e688857a1441cfb9d62bb0aae5e2ee as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/69e688857a1441cfb9d62bb0aae5e2ee 2024-11-08T02:30:40,362 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/69e688857a1441cfb9d62bb0aae5e2ee, entries=150, sequenceid=154, filesize=11.9 K 2024-11-08T02:30:40,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fea1c374518d472fa6470e99d9518d83 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fea1c374518d472fa6470e99d9518d83 2024-11-08T02:30:40,372 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fea1c374518d472fa6470e99d9518d83, entries=150, sequenceid=154, filesize=11.9 K 2024-11-08T02:30:40,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/9d2dbb80f70a463ca0c35bb79a531059 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/9d2dbb80f70a463ca0c35bb79a531059 2024-11-08T02:30:40,391 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/9d2dbb80f70a463ca0c35bb79a531059, entries=150, sequenceid=154, filesize=11.9 K 2024-11-08T02:30:40,415 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=20.13 KB/20610 for dc8ad63d9a6a95258392be1a735a16e9 in 179ms, sequenceid=154, compaction requested=true 2024-11-08T02:30:40,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-08T02:30:40,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:40,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17 2024-11-08T02:30:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=17 2024-11-08T02:30:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:40,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:30:40,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:40,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:40,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:40,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:40,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:40,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:40,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-08T02:30:40,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1070 sec 2024-11-08T02:30:40,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees in 1.1140 sec 2024-11-08T02:30:40,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/6884da23e7644bc8846904507bec8c5f is 50, key is test_row_0/A:col10/1731033040373/Put/seqid=0 2024-11-08T02:30:40,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741869_1045 (size=14541) 2024-11-08T02:30:40,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033100456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033100457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033100458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033100460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033100460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033100566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033100566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033100566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033100568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033100569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033100770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033100770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033100771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033100771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:40,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033100772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:40,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/6884da23e7644bc8846904507bec8c5f 2024-11-08T02:30:40,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e236aaa4806c4a01b05c1bfff3dfcf03 is 50, key is test_row_0/B:col10/1731033040373/Put/seqid=0 2024-11-08T02:30:40,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741870_1046 (size=12151) 2024-11-08T02:30:40,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e236aaa4806c4a01b05c1bfff3dfcf03 2024-11-08T02:30:40,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/ff9647b63a564c5eb8caf6ce6455db93 is 50, key is test_row_0/C:col10/1731033040373/Put/seqid=0 2024-11-08T02:30:40,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741871_1047 (size=12151) 2024-11-08T02:30:41,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033101075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033101076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033101076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033101076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033101077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/ff9647b63a564c5eb8caf6ce6455db93 2024-11-08T02:30:41,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/6884da23e7644bc8846904507bec8c5f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/6884da23e7644bc8846904507bec8c5f 2024-11-08T02:30:41,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/6884da23e7644bc8846904507bec8c5f, entries=200, sequenceid=165, filesize=14.2 K 2024-11-08T02:30:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e236aaa4806c4a01b05c1bfff3dfcf03 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e236aaa4806c4a01b05c1bfff3dfcf03 2024-11-08T02:30:41,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-08T02:30:41,416 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 16 completed 2024-11-08T02:30:41,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:41,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees 2024-11-08T02:30:41,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T02:30:41,424 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:41,425 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:41,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:41,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e236aaa4806c4a01b05c1bfff3dfcf03, entries=150, sequenceid=165, filesize=11.9 K 2024-11-08T02:30:41,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/ff9647b63a564c5eb8caf6ce6455db93 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/ff9647b63a564c5eb8caf6ce6455db93 2024-11-08T02:30:41,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/ff9647b63a564c5eb8caf6ce6455db93, entries=150, sequenceid=165, filesize=11.9 K 2024-11-08T02:30:41,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for dc8ad63d9a6a95258392be1a735a16e9 in 1033ms, sequenceid=165, compaction requested=true 2024-11-08T02:30:41,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:41,452 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:30:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:41,452 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:30:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:41,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:41,454 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50934 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:30:41,454 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:30:41,454 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:41,454 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:41,455 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:41,455 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7c63a5dbdf1f4b32b26bd1faafbaf9fe, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/060a7fb2ad2346b1bb7e0bac7750df24, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fea1c374518d472fa6470e99d9518d83, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e236aaa4806c4a01b05c1bfff3dfcf03] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=47.4 K 2024-11-08T02:30:41,455 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:41,456 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f784536f74524ae8a1b418a70a890dc2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/1e132f0f974c4f9590fe439a676cb93d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/69e688857a1441cfb9d62bb0aae5e2ee, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/6884da23e7644bc8846904507bec8c5f] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=49.7 K 2024-11-08T02:30:41,456 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 7c63a5dbdf1f4b32b26bd1faafbaf9fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033038176 2024-11-08T02:30:41,457 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f784536f74524ae8a1b418a70a890dc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033038176 2024-11-08T02:30:41,457 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 060a7fb2ad2346b1bb7e0bac7750df24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731033038519 2024-11-08T02:30:41,458 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting fea1c374518d472fa6470e99d9518d83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731033039234 2024-11-08T02:30:41,459 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e236aaa4806c4a01b05c1bfff3dfcf03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731033040373 2024-11-08T02:30:41,460 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1e132f0f974c4f9590fe439a676cb93d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731033038519 2024-11-08T02:30:41,461 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 69e688857a1441cfb9d62bb0aae5e2ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731033039234 2024-11-08T02:30:41,463 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6884da23e7644bc8846904507bec8c5f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731033040373 2024-11-08T02:30:41,485 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:41,486 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/14c696e97c264803aaee2c9bb2c3de64 is 50, key is test_row_0/A:col10/1731033040373/Put/seqid=0 2024-11-08T02:30:41,488 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:41,489 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e36806a0912b4ac38ce6a5024abacdf4 is 50, key is test_row_0/B:col10/1731033040373/Put/seqid=0 2024-11-08T02:30:41,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741872_1048 (size=12527) 2024-11-08T02:30:41,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741873_1049 (size=12527) 2024-11-08T02:30:41,523 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/14c696e97c264803aaee2c9bb2c3de64 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/14c696e97c264803aaee2c9bb2c3de64 2024-11-08T02:30:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T02:30:41,533 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into 14c696e97c264803aaee2c9bb2c3de64(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:41,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:41,533 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=12, startTime=1731033041452; duration=0sec 2024-11-08T02:30:41,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:41,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:41,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:30:41,536 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48544 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:30:41,536 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:41,536 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:41,536 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1c544aaeefcc4d94b658030f624691fb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/955778f4ac624705970d028afa4b2815, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/9d2dbb80f70a463ca0c35bb79a531059, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/ff9647b63a564c5eb8caf6ce6455db93] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=47.4 K 2024-11-08T02:30:41,538 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1c544aaeefcc4d94b658030f624691fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033038176 2024-11-08T02:30:41,539 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 955778f4ac624705970d028afa4b2815, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731033038519 2024-11-08T02:30:41,539 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9d2dbb80f70a463ca0c35bb79a531059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731033039234 2024-11-08T02:30:41,539 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ff9647b63a564c5eb8caf6ce6455db93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731033040373 2024-11-08T02:30:41,554 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:41,556 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6515238286e149a4b10de0b868d28df0 is 50, key is test_row_0/C:col10/1731033040373/Put/seqid=0 2024-11-08T02:30:41,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741874_1050 (size=12527) 2024-11-08T02:30:41,575 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6515238286e149a4b10de0b868d28df0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6515238286e149a4b10de0b868d28df0 2024-11-08T02:30:41,577 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-08T02:30:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:41,578 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:30:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:41,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:41,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:41,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:41,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/84a13b389fc342a68fce3d758f90cedf is 50, key is test_row_0/A:col10/1731033040456/Put/seqid=0 2024-11-08T02:30:41,586 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 6515238286e149a4b10de0b868d28df0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:41,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:41,586 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:41,586 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=12, startTime=1731033041452; duration=0sec 2024-11-08T02:30:41,586 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:41,586 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:41,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033101588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033101587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033101591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033101592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033101593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741875_1051 (size=12151) 2024-11-08T02:30:41,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033101695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033101696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033101696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033101697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033101697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T02:30:41,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033101897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033101900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033101901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033101901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:41,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033101901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:41,911 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e36806a0912b4ac38ce6a5024abacdf4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e36806a0912b4ac38ce6a5024abacdf4 2024-11-08T02:30:41,919 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into e36806a0912b4ac38ce6a5024abacdf4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:41,919 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:41,919 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=12, startTime=1731033041452; duration=0sec 2024-11-08T02:30:41,919 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:41,919 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:41,998 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/84a13b389fc342a68fce3d758f90cedf 2024-11-08T02:30:42,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/77eeb19648124bfe880e378ad48e23f7 is 50, key is test_row_0/B:col10/1731033040456/Put/seqid=0 2024-11-08T02:30:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T02:30:42,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741876_1052 (size=12151) 2024-11-08T02:30:42,034 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/77eeb19648124bfe880e378ad48e23f7 2024-11-08T02:30:42,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/e7fbdb3fc1cb434992604dbce35888c9 is 50, key is test_row_0/C:col10/1731033040456/Put/seqid=0 2024-11-08T02:30:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741877_1053 (size=12151) 2024-11-08T02:30:42,077 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/e7fbdb3fc1cb434992604dbce35888c9 2024-11-08T02:30:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/84a13b389fc342a68fce3d758f90cedf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/84a13b389fc342a68fce3d758f90cedf 2024-11-08T02:30:42,098 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/84a13b389fc342a68fce3d758f90cedf, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T02:30:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/77eeb19648124bfe880e378ad48e23f7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/77eeb19648124bfe880e378ad48e23f7 2024-11-08T02:30:42,109 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/77eeb19648124bfe880e378ad48e23f7, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T02:30:42,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/e7fbdb3fc1cb434992604dbce35888c9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e7fbdb3fc1cb434992604dbce35888c9 2024-11-08T02:30:42,145 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e7fbdb3fc1cb434992604dbce35888c9, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T02:30:42,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for dc8ad63d9a6a95258392be1a735a16e9 in 569ms, sequenceid=192, compaction requested=false 2024-11-08T02:30:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-08T02:30:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=19 2024-11-08T02:30:42,152 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-08T02:30:42,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 725 msec 2024-11-08T02:30:42,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees in 734 msec 2024-11-08T02:30:42,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:42,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:30:42,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:42,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:42,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:42,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:42,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:42,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:42,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/21a1ecd7aef243abb711cb683c7eb513 is 50, key is test_row_0/A:col10/1731033041591/Put/seqid=0 2024-11-08T02:30:42,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741878_1054 (size=14541) 2024-11-08T02:30:42,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/21a1ecd7aef243abb711cb683c7eb513 2024-11-08T02:30:42,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fc60bda880554edf92004fd4a6520db2 is 50, key is test_row_0/B:col10/1731033041591/Put/seqid=0 2024-11-08T02:30:42,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033102234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033102238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741879_1055 (size=12151) 2024-11-08T02:30:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033102241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033102241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033102243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fc60bda880554edf92004fd4a6520db2 2024-11-08T02:30:42,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/94ebabbb2fd84be1b67ac1841c52cb10 is 50, key is test_row_0/C:col10/1731033041591/Put/seqid=0 2024-11-08T02:30:42,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741880_1056 (size=12151) 2024-11-08T02:30:42,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033102343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,344 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033102343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033102348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033102348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033102349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-08T02:30:42,527 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 18 completed 2024-11-08T02:30:42,528 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees 2024-11-08T02:30:42,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-08T02:30:42,530 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:42,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:42,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:42,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033102545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033102546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033102551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033102552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033102552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-08T02:30:42,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/94ebabbb2fd84be1b67ac1841c52cb10 2024-11-08T02:30:42,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/21a1ecd7aef243abb711cb683c7eb513 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/21a1ecd7aef243abb711cb683c7eb513 2024-11-08T02:30:42,682 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-08T02:30:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:42,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:42,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:42,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/21a1ecd7aef243abb711cb683c7eb513, entries=200, sequenceid=205, filesize=14.2 K 2024-11-08T02:30:42,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fc60bda880554edf92004fd4a6520db2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fc60bda880554edf92004fd4a6520db2 2024-11-08T02:30:42,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fc60bda880554edf92004fd4a6520db2, entries=150, sequenceid=205, filesize=11.9 K 2024-11-08T02:30:42,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/94ebabbb2fd84be1b67ac1841c52cb10 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/94ebabbb2fd84be1b67ac1841c52cb10 2024-11-08T02:30:42,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/94ebabbb2fd84be1b67ac1841c52cb10, entries=150, sequenceid=205, filesize=11.9 K 2024-11-08T02:30:42,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for dc8ad63d9a6a95258392be1a735a16e9 in 527ms, sequenceid=205, compaction requested=true 2024-11-08T02:30:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:42,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:42,730 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:42,730 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:42,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:42,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:42,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:42,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:42,732 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:42,732 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:42,732 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:42,732 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e36806a0912b4ac38ce6a5024abacdf4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/77eeb19648124bfe880e378ad48e23f7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fc60bda880554edf92004fd4a6520db2] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.0 K 2024-11-08T02:30:42,733 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:42,734 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e36806a0912b4ac38ce6a5024abacdf4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731033040373 2024-11-08T02:30:42,734 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 77eeb19648124bfe880e378ad48e23f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731033040456 2024-11-08T02:30:42,735 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting fc60bda880554edf92004fd4a6520db2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731033041591 2024-11-08T02:30:42,736 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:42,736 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:42,736 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/14c696e97c264803aaee2c9bb2c3de64, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/84a13b389fc342a68fce3d758f90cedf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/21a1ecd7aef243abb711cb683c7eb513] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=38.3 K 2024-11-08T02:30:42,738 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 14c696e97c264803aaee2c9bb2c3de64, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731033040373 2024-11-08T02:30:42,738 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 84a13b389fc342a68fce3d758f90cedf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731033040456 2024-11-08T02:30:42,739 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 21a1ecd7aef243abb711cb683c7eb513, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731033041584 2024-11-08T02:30:42,751 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:42,752 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e54d0f37a8d5480b8315d48b1566166d is 50, key is test_row_0/B:col10/1731033041591/Put/seqid=0 2024-11-08T02:30:42,754 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:42,754 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/7e88852277354ac89b3e5c9b67ce56c7 is 50, key is test_row_0/A:col10/1731033041591/Put/seqid=0 2024-11-08T02:30:42,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741881_1057 (size=12629) 2024-11-08T02:30:42,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741882_1058 (size=12629) 2024-11-08T02:30:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-08T02:30:42,836 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-08T02:30:42,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:42,837 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T02:30:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/3cde3978e2524cd0b09ebc985c725fe5 is 50, key is test_row_0/A:col10/1731033042241/Put/seqid=0 2024-11-08T02:30:42,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741883_1059 (size=12151) 2024-11-08T02:30:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:42,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033102858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033102860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033102861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033102862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033102866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033102967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033102967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033102969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:42,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033102970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-08T02:30:43,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033103170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033103170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033103173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033103173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,184 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e54d0f37a8d5480b8315d48b1566166d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e54d0f37a8d5480b8315d48b1566166d 2024-11-08T02:30:43,190 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/7e88852277354ac89b3e5c9b67ce56c7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/7e88852277354ac89b3e5c9b67ce56c7 2024-11-08T02:30:43,196 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into e54d0f37a8d5480b8315d48b1566166d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:43,196 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:43,196 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033042730; duration=0sec 2024-11-08T02:30:43,197 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:43,197 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:43,197 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:43,200 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into 7e88852277354ac89b3e5c9b67ce56c7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:43,200 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:43,200 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033042729; duration=0sec 2024-11-08T02:30:43,200 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:43,200 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:43,201 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:43,201 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:43,202 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:43,202 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6515238286e149a4b10de0b868d28df0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e7fbdb3fc1cb434992604dbce35888c9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/94ebabbb2fd84be1b67ac1841c52cb10] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.0 K 2024-11-08T02:30:43,202 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6515238286e149a4b10de0b868d28df0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731033040373 2024-11-08T02:30:43,203 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e7fbdb3fc1cb434992604dbce35888c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731033040456 2024-11-08T02:30:43,204 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 94ebabbb2fd84be1b67ac1841c52cb10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731033041591 2024-11-08T02:30:43,212 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#45 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:43,212 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/0d63cafa289940e7802693db3f520ec0 is 50, key is test_row_0/C:col10/1731033041591/Put/seqid=0 2024-11-08T02:30:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741884_1060 (size=12629) 2024-11-08T02:30:43,232 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/0d63cafa289940e7802693db3f520ec0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/0d63cafa289940e7802693db3f520ec0 2024-11-08T02:30:43,241 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 0d63cafa289940e7802693db3f520ec0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:43,241 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:43,241 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033042730; duration=0sec 2024-11-08T02:30:43,241 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:43,241 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:43,248 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/3cde3978e2524cd0b09ebc985c725fe5 2024-11-08T02:30:43,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/041e4001806f4275817051c7d13c2728 is 50, key is test_row_0/B:col10/1731033042241/Put/seqid=0 2024-11-08T02:30:43,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741885_1061 (size=12151) 2024-11-08T02:30:43,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033103370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033103473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033103477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033103484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033103485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-08T02:30:43,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/041e4001806f4275817051c7d13c2728 2024-11-08T02:30:43,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/8fe03ff735b64102a47cc0395662c18d is 50, key is test_row_0/C:col10/1731033042241/Put/seqid=0 2024-11-08T02:30:43,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741886_1062 (size=12151) 2024-11-08T02:30:43,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033103976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033103982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033103986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:43,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:43,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033103990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,100 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/8fe03ff735b64102a47cc0395662c18d 2024-11-08T02:30:44,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/3cde3978e2524cd0b09ebc985c725fe5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/3cde3978e2524cd0b09ebc985c725fe5 2024-11-08T02:30:44,117 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/3cde3978e2524cd0b09ebc985c725fe5, entries=150, sequenceid=231, filesize=11.9 K 2024-11-08T02:30:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/041e4001806f4275817051c7d13c2728 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/041e4001806f4275817051c7d13c2728 2024-11-08T02:30:44,125 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/041e4001806f4275817051c7d13c2728, entries=150, sequenceid=231, filesize=11.9 K 2024-11-08T02:30:44,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/8fe03ff735b64102a47cc0395662c18d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/8fe03ff735b64102a47cc0395662c18d 2024-11-08T02:30:44,135 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/8fe03ff735b64102a47cc0395662c18d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-08T02:30:44,136 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for dc8ad63d9a6a95258392be1a735a16e9 in 1299ms, sequenceid=231, compaction requested=false 2024-11-08T02:30:44,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:44,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:44,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21 2024-11-08T02:30:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=21 2024-11-08T02:30:44,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20 2024-11-08T02:30:44,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2024-11-08T02:30:44,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees in 1.6120 sec 2024-11-08T02:30:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:44,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:30:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:44,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:44,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b6ed57a064d8496991d23ce9ce0f5dc1 is 50, key is test_row_0/A:col10/1731033042861/Put/seqid=0 2024-11-08T02:30:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741887_1063 (size=12151) 2024-11-08T02:30:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033104444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:44,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033104546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-08T02:30:44,634 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 20 completed 2024-11-08T02:30:44,635 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees 2024-11-08T02:30:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-08T02:30:44,637 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:44,637 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=22, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:44,637 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-08T02:30:44,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033104749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,789 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-08T02:30:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:44,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:44,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:44,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b6ed57a064d8496991d23ce9ce0f5dc1 2024-11-08T02:30:44,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/97a3d87bf8fe492ab15ff3f107c2c7b8 is 50, key is test_row_0/B:col10/1731033042861/Put/seqid=0 2024-11-08T02:30:44,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741888_1064 (size=12151) 2024-11-08T02:30:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-08T02:30:44,942 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-08T02:30:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:44,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:44,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:44,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:44,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033104984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:44,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033104987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:44,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:44,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033104997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:45,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033104999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033105054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,096 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-08T02:30:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/97a3d87bf8fe492ab15ff3f107c2c7b8 2024-11-08T02:30:45,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-08T02:30:45,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/b3963743e23c42f3bf26d24f93469909 is 50, key is test_row_0/C:col10/1731033042861/Put/seqid=0 2024-11-08T02:30:45,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741889_1065 (size=12151) 2024-11-08T02:30:45,250 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-08T02:30:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,403 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-08T02:30:45,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:45,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,556 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-08T02:30:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:45,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:45,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033105558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/b3963743e23c42f3bf26d24f93469909 2024-11-08T02:30:45,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b6ed57a064d8496991d23ce9ce0f5dc1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b6ed57a064d8496991d23ce9ce0f5dc1 2024-11-08T02:30:45,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b6ed57a064d8496991d23ce9ce0f5dc1, entries=150, sequenceid=246, filesize=11.9 K 2024-11-08T02:30:45,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/97a3d87bf8fe492ab15ff3f107c2c7b8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/97a3d87bf8fe492ab15ff3f107c2c7b8 2024-11-08T02:30:45,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/97a3d87bf8fe492ab15ff3f107c2c7b8, entries=150, sequenceid=246, filesize=11.9 K 2024-11-08T02:30:45,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/b3963743e23c42f3bf26d24f93469909 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b3963743e23c42f3bf26d24f93469909 2024-11-08T02:30:45,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b3963743e23c42f3bf26d24f93469909, entries=150, sequenceid=246, filesize=11.9 K 2024-11-08T02:30:45,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dc8ad63d9a6a95258392be1a735a16e9 in 1298ms, sequenceid=246, compaction requested=true 2024-11-08T02:30:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:45,680 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:45,680 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:45,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:45,681 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:45,681 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:45,681 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:45,681 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:45,681 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,681 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,682 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e54d0f37a8d5480b8315d48b1566166d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/041e4001806f4275817051c7d13c2728, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/97a3d87bf8fe492ab15ff3f107c2c7b8] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.1 K 2024-11-08T02:30:45,682 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/7e88852277354ac89b3e5c9b67ce56c7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/3cde3978e2524cd0b09ebc985c725fe5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b6ed57a064d8496991d23ce9ce0f5dc1] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.1 K 2024-11-08T02:30:45,682 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7e88852277354ac89b3e5c9b67ce56c7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731033041591 2024-11-08T02:30:45,682 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e54d0f37a8d5480b8315d48b1566166d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731033041591 2024-11-08T02:30:45,682 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3cde3978e2524cd0b09ebc985c725fe5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731033042232 2024-11-08T02:30:45,682 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 041e4001806f4275817051c7d13c2728, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731033042232 2024-11-08T02:30:45,683 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b6ed57a064d8496991d23ce9ce0f5dc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731033042861 2024-11-08T02:30:45,683 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 97a3d87bf8fe492ab15ff3f107c2c7b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731033042861 2024-11-08T02:30:45,691 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:45,692 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:45,692 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/ef3b096ef602409a93dcac074b6ba8bf is 50, key is test_row_0/A:col10/1731033042861/Put/seqid=0 2024-11-08T02:30:45,692 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/dec1c0b4e9324022a27677f9a4ed04b8 is 50, key is test_row_0/B:col10/1731033042861/Put/seqid=0 2024-11-08T02:30:45,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741891_1067 (size=12731) 2024-11-08T02:30:45,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741890_1066 (size=12731) 2024-11-08T02:30:45,710 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:45,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-08T02:30:45,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:30:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:45,719 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/ef3b096ef602409a93dcac074b6ba8bf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/ef3b096ef602409a93dcac074b6ba8bf 2024-11-08T02:30:45,727 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into ef3b096ef602409a93dcac074b6ba8bf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:45,727 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:45,727 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033045679; duration=0sec 2024-11-08T02:30:45,727 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:45,727 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:45,728 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:45,729 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:45,729 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:45,729 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:45,730 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/0d63cafa289940e7802693db3f520ec0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/8fe03ff735b64102a47cc0395662c18d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b3963743e23c42f3bf26d24f93469909] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.1 K 2024-11-08T02:30:45,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8a6758b0674d4ccd869d185811bd248b is 50, key is test_row_0/A:col10/1731033044441/Put/seqid=0 2024-11-08T02:30:45,734 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0d63cafa289940e7802693db3f520ec0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731033041591 2024-11-08T02:30:45,734 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8fe03ff735b64102a47cc0395662c18d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731033042232 2024-11-08T02:30:45,735 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b3963743e23c42f3bf26d24f93469909, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731033042861 2024-11-08T02:30:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-08T02:30:45,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741892_1068 (size=12301) 2024-11-08T02:30:45,745 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8a6758b0674d4ccd869d185811bd248b 2024-11-08T02:30:45,751 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:45,752 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/f46cbe4866c64fba926dd3eb6a4ff8fb is 50, key is test_row_0/C:col10/1731033042861/Put/seqid=0 2024-11-08T02:30:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/2c614466f2c14138ab4d0827038b3842 is 50, key is test_row_0/B:col10/1731033044441/Put/seqid=0 2024-11-08T02:30:45,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741893_1069 (size=12731) 2024-11-08T02:30:45,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741894_1070 (size=12301) 2024-11-08T02:30:46,114 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/dec1c0b4e9324022a27677f9a4ed04b8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/dec1c0b4e9324022a27677f9a4ed04b8 2024-11-08T02:30:46,122 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into dec1c0b4e9324022a27677f9a4ed04b8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:46,122 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:46,122 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033045680; duration=0sec 2024-11-08T02:30:46,123 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:46,123 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:46,179 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/2c614466f2c14138ab4d0827038b3842 2024-11-08T02:30:46,184 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/f46cbe4866c64fba926dd3eb6a4ff8fb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f46cbe4866c64fba926dd3eb6a4ff8fb 2024-11-08T02:30:46,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/fac02c2c50924e8da643285bcd6f2c57 is 50, key is test_row_0/C:col10/1731033044441/Put/seqid=0 2024-11-08T02:30:46,194 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into f46cbe4866c64fba926dd3eb6a4ff8fb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:46,194 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:46,194 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033045680; duration=0sec 2024-11-08T02:30:46,194 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:46,194 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:46,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741895_1071 (size=12301) 2024-11-08T02:30:46,209 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/fac02c2c50924e8da643285bcd6f2c57 2024-11-08T02:30:46,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8a6758b0674d4ccd869d185811bd248b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8a6758b0674d4ccd869d185811bd248b 2024-11-08T02:30:46,224 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8a6758b0674d4ccd869d185811bd248b, entries=150, sequenceid=270, filesize=12.0 K 2024-11-08T02:30:46,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/2c614466f2c14138ab4d0827038b3842 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/2c614466f2c14138ab4d0827038b3842 2024-11-08T02:30:46,239 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/2c614466f2c14138ab4d0827038b3842, entries=150, sequenceid=270, filesize=12.0 K 2024-11-08T02:30:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/fac02c2c50924e8da643285bcd6f2c57 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/fac02c2c50924e8da643285bcd6f2c57 2024-11-08T02:30:46,249 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/fac02c2c50924e8da643285bcd6f2c57, entries=150, sequenceid=270, filesize=12.0 K 2024-11-08T02:30:46,252 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for dc8ad63d9a6a95258392be1a735a16e9 in 541ms, sequenceid=270, compaction requested=false 2024-11-08T02:30:46,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:46,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:46,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23 2024-11-08T02:30:46,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=23 2024-11-08T02:30:46,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-08T02:30:46,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-08T02:30:46,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees in 1.6240 sec 2024-11-08T02:30:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:46,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:30:46,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:46,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:46,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:46,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:46,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:46,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:46,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8232298bf6e14c5aaa9375c956cbc828 is 50, key is test_row_0/A:col10/1731033046587/Put/seqid=0 2024-11-08T02:30:46,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741896_1072 (size=14741) 2024-11-08T02:30:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033106653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-08T02:30:46,741 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 22 completed 2024-11-08T02:30:46,743 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees 2024-11-08T02:30:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T02:30:46,746 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:46,746 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=24, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:46,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:46,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033106755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:46,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T02:30:46,898 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:46,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-08T02:30:46,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:46,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:46,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:46,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:46,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:46,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:46,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033106961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033106995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,001 DEBUG [Thread-149 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4140 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:30:47,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8232298bf6e14c5aaa9375c956cbc828 2024-11-08T02:30:47,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033107005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033107005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,008 DEBUG [Thread-157 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4149 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:30:47,008 DEBUG [Thread-153 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4143 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:30:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033107014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,016 DEBUG [Thread-155 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4154 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:30:47,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/70bd1abe1b504eb6a46da755b9cd2146 is 50, key is test_row_0/B:col10/1731033046587/Put/seqid=0 2024-11-08T02:30:47,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741897_1073 (size=12301) 2024-11-08T02:30:47,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/70bd1abe1b504eb6a46da755b9cd2146 2024-11-08T02:30:47,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/09a0f4a4be894aeba47a46da4bc5ce51 is 50, key is test_row_0/C:col10/1731033046587/Put/seqid=0 2024-11-08T02:30:47,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T02:30:47,051 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-08T02:30:47,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:47,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741898_1074 (size=12301) 2024-11-08T02:30:47,206 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-08T02:30:47,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:47,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033107266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T02:30:47,359 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-08T02:30:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/09a0f4a4be894aeba47a46da4bc5ce51 2024-11-08T02:30:47,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8232298bf6e14c5aaa9375c956cbc828 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8232298bf6e14c5aaa9375c956cbc828 2024-11-08T02:30:47,512 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-08T02:30:47,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:47,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8232298bf6e14c5aaa9375c956cbc828, entries=200, sequenceid=284, filesize=14.4 K 2024-11-08T02:30:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:47,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/70bd1abe1b504eb6a46da755b9cd2146 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/70bd1abe1b504eb6a46da755b9cd2146 2024-11-08T02:30:47,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/70bd1abe1b504eb6a46da755b9cd2146, entries=150, sequenceid=284, filesize=12.0 K 2024-11-08T02:30:47,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/09a0f4a4be894aeba47a46da4bc5ce51 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/09a0f4a4be894aeba47a46da4bc5ce51 2024-11-08T02:30:47,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/09a0f4a4be894aeba47a46da4bc5ce51, entries=150, sequenceid=284, filesize=12.0 K 2024-11-08T02:30:47,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for dc8ad63d9a6a95258392be1a735a16e9 in 943ms, sequenceid=284, compaction requested=true 2024-11-08T02:30:47,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:47,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:47,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:47,532 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:47,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:47,532 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:47,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:47,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:47,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:47,533 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:47,533 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:47,533 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,533 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/dec1c0b4e9324022a27677f9a4ed04b8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/2c614466f2c14138ab4d0827038b3842, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/70bd1abe1b504eb6a46da755b9cd2146] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.5 K 2024-11-08T02:30:47,534 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:47,534 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:47,534 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,534 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/ef3b096ef602409a93dcac074b6ba8bf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8a6758b0674d4ccd869d185811bd248b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8232298bf6e14c5aaa9375c956cbc828] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=38.8 K 2024-11-08T02:30:47,534 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting dec1c0b4e9324022a27677f9a4ed04b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731033042861 2024-11-08T02:30:47,534 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ef3b096ef602409a93dcac074b6ba8bf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731033042861 2024-11-08T02:30:47,535 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 2c614466f2c14138ab4d0827038b3842, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731033044421 2024-11-08T02:30:47,535 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8a6758b0674d4ccd869d185811bd248b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731033044421 2024-11-08T02:30:47,535 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 70bd1abe1b504eb6a46da755b9cd2146, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731033046582 2024-11-08T02:30:47,535 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8232298bf6e14c5aaa9375c956cbc828, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731033046579 2024-11-08T02:30:47,548 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:47,548 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:47,549 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fcf0d486e6d14735841e61685c87ed34 is 50, key is test_row_0/B:col10/1731033046587/Put/seqid=0 2024-11-08T02:30:47,549 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/45e30d16a45a426db7fcb6245b01773a is 50, key is test_row_0/A:col10/1731033046587/Put/seqid=0 2024-11-08T02:30:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741899_1075 (size=12983) 2024-11-08T02:30:47,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741900_1076 (size=12983) 2024-11-08T02:30:47,577 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/45e30d16a45a426db7fcb6245b01773a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/45e30d16a45a426db7fcb6245b01773a 2024-11-08T02:30:47,585 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into 45e30d16a45a426db7fcb6245b01773a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:47,585 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:47,585 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033047531; duration=0sec 2024-11-08T02:30:47,585 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:47,586 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:47,586 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:47,587 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:47,587 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:47,587 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,587 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f46cbe4866c64fba926dd3eb6a4ff8fb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/fac02c2c50924e8da643285bcd6f2c57, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/09a0f4a4be894aeba47a46da4bc5ce51] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.5 K 2024-11-08T02:30:47,588 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f46cbe4866c64fba926dd3eb6a4ff8fb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731033042861 2024-11-08T02:30:47,588 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting fac02c2c50924e8da643285bcd6f2c57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731033044421 2024-11-08T02:30:47,589 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 09a0f4a4be894aeba47a46da4bc5ce51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731033046582 2024-11-08T02:30:47,597 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:47,597 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/40fd639b5f14437b9e1cbb2bd98b9244 is 50, key is test_row_0/C:col10/1731033046587/Put/seqid=0 2024-11-08T02:30:47,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741901_1077 (size=12983) 2024-11-08T02:30:47,665 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-08T02:30:47,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:47,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:30:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:47,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:47,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/700f9c6328754a038287308fc3427625 is 50, key is test_row_0/A:col10/1731033046649/Put/seqid=0 2024-11-08T02:30:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741902_1078 (size=12301) 2024-11-08T02:30:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:47,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:47,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033107790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T02:30:47,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033107893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:47,972 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/fcf0d486e6d14735841e61685c87ed34 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fcf0d486e6d14735841e61685c87ed34 2024-11-08T02:30:47,977 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into fcf0d486e6d14735841e61685c87ed34(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:47,977 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:47,977 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033047532; duration=0sec 2024-11-08T02:30:47,978 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:47,978 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:48,012 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/40fd639b5f14437b9e1cbb2bd98b9244 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/40fd639b5f14437b9e1cbb2bd98b9244 2024-11-08T02:30:48,020 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 40fd639b5f14437b9e1cbb2bd98b9244(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:48,020 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:48,020 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033047532; duration=0sec 2024-11-08T02:30:48,020 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:48,020 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:48,084 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/700f9c6328754a038287308fc3427625 2024-11-08T02:30:48,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/d425c40746e248c4b1fdb1848a98267e is 50, key is test_row_0/B:col10/1731033046649/Put/seqid=0 2024-11-08T02:30:48,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033108096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:48,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741903_1079 (size=12301) 2024-11-08T02:30:48,106 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/d425c40746e248c4b1fdb1848a98267e 2024-11-08T02:30:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c is 50, key is test_row_0/C:col10/1731033046649/Put/seqid=0 2024-11-08T02:30:48,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741904_1080 (size=12301) 2024-11-08T02:30:48,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033108401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:48,525 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c 2024-11-08T02:30:48,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/700f9c6328754a038287308fc3427625 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/700f9c6328754a038287308fc3427625 2024-11-08T02:30:48,538 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/700f9c6328754a038287308fc3427625, entries=150, sequenceid=310, filesize=12.0 K 2024-11-08T02:30:48,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/d425c40746e248c4b1fdb1848a98267e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d425c40746e248c4b1fdb1848a98267e 2024-11-08T02:30:48,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d425c40746e248c4b1fdb1848a98267e, entries=150, sequenceid=310, filesize=12.0 K 2024-11-08T02:30:48,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c 2024-11-08T02:30:48,555 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c, entries=150, sequenceid=310, filesize=12.0 K 2024-11-08T02:30:48,556 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for dc8ad63d9a6a95258392be1a735a16e9 in 890ms, sequenceid=310, compaction requested=false 2024-11-08T02:30:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:48,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:48,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-11-08T02:30:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-11-08T02:30:48,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-08T02:30:48,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8120 sec 2024-11-08T02:30:48,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees in 1.8170 sec 2024-11-08T02:30:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-08T02:30:48,854 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 24 completed 2024-11-08T02:30:48,855 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees 2024-11-08T02:30:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-08T02:30:48,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:48,857 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=26, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:48,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:48,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:30:48,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:48,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:48,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:48,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:48,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:48,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:48,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/74a36d08f5384a6e91ef4622bc58efe5 is 50, key is test_row_0/A:col10/1731033048906/Put/seqid=0 2024-11-08T02:30:48,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741905_1081 (size=14741) 2024-11-08T02:30:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-08T02:30:48,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:48,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033108960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,008 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-08T02:30:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033109062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-08T02:30:49,161 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-08T02:30:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033109264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,314 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-08T02:30:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/74a36d08f5384a6e91ef4622bc58efe5 2024-11-08T02:30:49,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/740fa2c3a7b347da9d5131703ba19f87 is 50, key is test_row_0/B:col10/1731033048906/Put/seqid=0 2024-11-08T02:30:49,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741906_1082 (size=12301) 2024-11-08T02:30:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-08T02:30:49,467 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-08T02:30:49,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:49,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033109566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,620 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-08T02:30:49,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:49,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/740fa2c3a7b347da9d5131703ba19f87 2024-11-08T02:30:49,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/c336c88034fc4fbaa9193522e453408b is 50, key is test_row_0/C:col10/1731033048906/Put/seqid=0 2024-11-08T02:30:49,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741907_1083 (size=12301) 2024-11-08T02:30:49,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/c336c88034fc4fbaa9193522e453408b 2024-11-08T02:30:49,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/74a36d08f5384a6e91ef4622bc58efe5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/74a36d08f5384a6e91ef4622bc58efe5 2024-11-08T02:30:49,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/74a36d08f5384a6e91ef4622bc58efe5, entries=200, sequenceid=324, filesize=14.4 K 2024-11-08T02:30:49,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/740fa2c3a7b347da9d5131703ba19f87 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/740fa2c3a7b347da9d5131703ba19f87 2024-11-08T02:30:49,773 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-08T02:30:49,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:49,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:49,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/740fa2c3a7b347da9d5131703ba19f87, entries=150, sequenceid=324, filesize=12.0 K 2024-11-08T02:30:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/c336c88034fc4fbaa9193522e453408b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c336c88034fc4fbaa9193522e453408b 2024-11-08T02:30:49,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c336c88034fc4fbaa9193522e453408b, entries=150, sequenceid=324, filesize=12.0 K 2024-11-08T02:30:49,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for dc8ad63d9a6a95258392be1a735a16e9 in 883ms, sequenceid=324, compaction requested=true 2024-11-08T02:30:49,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:49,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:49,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:49,790 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:49,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:49,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:49,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:49,790 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:49,791 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:49,793 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:49,793 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:49,793 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,793 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fcf0d486e6d14735841e61685c87ed34, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d425c40746e248c4b1fdb1848a98267e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/740fa2c3a7b347da9d5131703ba19f87] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.7 K 2024-11-08T02:30:49,793 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:49,793 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:49,794 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,794 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/45e30d16a45a426db7fcb6245b01773a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/700f9c6328754a038287308fc3427625, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/74a36d08f5384a6e91ef4622bc58efe5] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=39.1 K 2024-11-08T02:30:49,794 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting fcf0d486e6d14735841e61685c87ed34, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731033046582 2024-11-08T02:30:49,795 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 45e30d16a45a426db7fcb6245b01773a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731033046582 2024-11-08T02:30:49,795 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting d425c40746e248c4b1fdb1848a98267e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731033046639 2024-11-08T02:30:49,795 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 700f9c6328754a038287308fc3427625, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731033046639 2024-11-08T02:30:49,796 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 74a36d08f5384a6e91ef4622bc58efe5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731033047782 2024-11-08T02:30:49,796 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 740fa2c3a7b347da9d5131703ba19f87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731033047785 2024-11-08T02:30:49,809 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:49,809 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/842592239c084abdb1dfb9f614741d27 is 50, key is test_row_0/B:col10/1731033048906/Put/seqid=0 2024-11-08T02:30:49,813 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:49,813 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/4e3aaadbabdf44709db7385dae253f65 is 50, key is test_row_0/A:col10/1731033048906/Put/seqid=0 2024-11-08T02:30:49,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741908_1084 (size=13085) 2024-11-08T02:30:49,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741909_1085 (size=13085) 2024-11-08T02:30:49,824 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/842592239c084abdb1dfb9f614741d27 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/842592239c084abdb1dfb9f614741d27 2024-11-08T02:30:49,833 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into 842592239c084abdb1dfb9f614741d27(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:49,833 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:49,833 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033049790; duration=0sec 2024-11-08T02:30:49,833 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:49,833 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:49,834 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:49,834 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/4e3aaadbabdf44709db7385dae253f65 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/4e3aaadbabdf44709db7385dae253f65 2024-11-08T02:30:49,835 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:49,835 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:49,836 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,836 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/40fd639b5f14437b9e1cbb2bd98b9244, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c336c88034fc4fbaa9193522e453408b] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.7 K 2024-11-08T02:30:49,836 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 40fd639b5f14437b9e1cbb2bd98b9244, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731033046582 2024-11-08T02:30:49,837 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b8af53f2f5f94f2bafb7d58f2d2c6f5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731033046639 2024-11-08T02:30:49,838 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c336c88034fc4fbaa9193522e453408b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731033047785 2024-11-08T02:30:49,844 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into 4e3aaadbabdf44709db7385dae253f65(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:49,844 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:49,844 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033049790; duration=0sec 2024-11-08T02:30:49,844 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:49,844 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:49,852 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:49,853 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/61812f9aa06344769879e77c1f86534f is 50, key is test_row_0/C:col10/1731033048906/Put/seqid=0 2024-11-08T02:30:49,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741910_1086 (size=13085) 2024-11-08T02:30:49,868 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/61812f9aa06344769879e77c1f86534f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/61812f9aa06344769879e77c1f86534f 2024-11-08T02:30:49,876 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 61812f9aa06344769879e77c1f86534f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:49,876 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:49,876 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033049790; duration=0sec 2024-11-08T02:30:49,876 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:49,876 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:49,926 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:49,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-08T02:30:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:49,927 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:30:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:49,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8e8e83eae1504899b7a9f40836593bef is 50, key is test_row_0/A:col10/1731033048953/Put/seqid=0 2024-11-08T02:30:49,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741911_1087 (size=12301) 2024-11-08T02:30:49,940 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8e8e83eae1504899b7a9f40836593bef 2024-11-08T02:30:49,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/55b52b8bcc004b7297f591dee5a14479 is 50, key is test_row_0/B:col10/1731033048953/Put/seqid=0 2024-11-08T02:30:49,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741912_1088 (size=12301) 2024-11-08T02:30:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-08T02:30:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:50,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:50,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:50,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033110093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033110196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:50,357 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/55b52b8bcc004b7297f591dee5a14479 2024-11-08T02:30:50,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/62b6d96c670a4670819bbe91dd31dd13 is 50, key is test_row_0/C:col10/1731033048953/Put/seqid=0 2024-11-08T02:30:50,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741913_1089 (size=12301) 2024-11-08T02:30:50,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:50,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033110400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033110702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:50,773 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/62b6d96c670a4670819bbe91dd31dd13 2024-11-08T02:30:50,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/8e8e83eae1504899b7a9f40836593bef as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e8e83eae1504899b7a9f40836593bef 2024-11-08T02:30:50,784 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e8e83eae1504899b7a9f40836593bef, entries=150, sequenceid=351, filesize=12.0 K 2024-11-08T02:30:50,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/55b52b8bcc004b7297f591dee5a14479 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55b52b8bcc004b7297f591dee5a14479 2024-11-08T02:30:50,790 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55b52b8bcc004b7297f591dee5a14479, entries=150, sequenceid=351, filesize=12.0 K 2024-11-08T02:30:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/62b6d96c670a4670819bbe91dd31dd13 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/62b6d96c670a4670819bbe91dd31dd13 2024-11-08T02:30:50,798 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/62b6d96c670a4670819bbe91dd31dd13, entries=150, sequenceid=351, filesize=12.0 K 2024-11-08T02:30:50,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for dc8ad63d9a6a95258392be1a735a16e9 in 872ms, sequenceid=351, compaction requested=false 2024-11-08T02:30:50,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:50,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:50,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=27 2024-11-08T02:30:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=27 2024-11-08T02:30:50,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=27, resume processing ppid=26 2024-11-08T02:30:50,802 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, ppid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-11-08T02:30:50,803 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees in 1.9470 sec 2024-11-08T02:30:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-08T02:30:50,960 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 26 completed 2024-11-08T02:30:50,961 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:30:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees 2024-11-08T02:30:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-08T02:30:50,963 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:30:50,963 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=28, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:30:50,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:30:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:51,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:30:51,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:51,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:51,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:51,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:51,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:51,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:51,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/a66ca2f19e7649c4a1e03ad69f413f16 is 50, key is test_row_0/A:col10/1731033051018/Put/seqid=0 2024-11-08T02:30:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741914_1090 (size=14741) 2024-11-08T02:30:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033111043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033111044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033111044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033111044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-08T02:30:51,114 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:51,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033111147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033111147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033111148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033111148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033111205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-08T02:30:51,268 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:51,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033111350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033111350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033111351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033111351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,420 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/a66ca2f19e7649c4a1e03ad69f413f16 2024-11-08T02:30:51,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/c68484d5f2e043119c541bab73579167 is 50, key is test_row_0/B:col10/1731033051018/Put/seqid=0 2024-11-08T02:30:51,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741915_1091 (size=12301) 2024-11-08T02:30:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-08T02:30:51,573 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033111653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033111653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033111654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033111654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,752 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/c68484d5f2e043119c541bab73579167 2024-11-08T02:30:51,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/bd922b1622974f7eaff17baefa554d2a is 50, key is test_row_0/C:col10/1731033051018/Put/seqid=0 2024-11-08T02:30:51,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741916_1092 (size=12301) 2024-11-08T02:30:51,904 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:51,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:51,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:51,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:52,057 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:52,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:52,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:52,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-08T02:30:52,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:52,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033112156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:52,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033112156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033112158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:52,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033112159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,209 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:52,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033112209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:52,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/bd922b1622974f7eaff17baefa554d2a 2024-11-08T02:30:52,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/a66ca2f19e7649c4a1e03ad69f413f16 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/a66ca2f19e7649c4a1e03ad69f413f16 2024-11-08T02:30:52,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/a66ca2f19e7649c4a1e03ad69f413f16, entries=200, sequenceid=365, filesize=14.4 K 2024-11-08T02:30:52,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/c68484d5f2e043119c541bab73579167 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c68484d5f2e043119c541bab73579167 2024-11-08T02:30:52,276 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c68484d5f2e043119c541bab73579167, entries=150, sequenceid=365, filesize=12.0 K 2024-11-08T02:30:52,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/bd922b1622974f7eaff17baefa554d2a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/bd922b1622974f7eaff17baefa554d2a 2024-11-08T02:30:52,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/bd922b1622974f7eaff17baefa554d2a, entries=150, sequenceid=365, filesize=12.0 K 2024-11-08T02:30:52,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for dc8ad63d9a6a95258392be1a735a16e9 in 1264ms, sequenceid=365, compaction requested=true 2024-11-08T02:30:52,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:52,284 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:52,284 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:52,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:52,285 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:52,285 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:52,285 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:52,285 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:52,285 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,285 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,285 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/4e3aaadbabdf44709db7385dae253f65, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e8e83eae1504899b7a9f40836593bef, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/a66ca2f19e7649c4a1e03ad69f413f16] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=39.2 K 2024-11-08T02:30:52,285 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/842592239c084abdb1dfb9f614741d27, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55b52b8bcc004b7297f591dee5a14479, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c68484d5f2e043119c541bab73579167] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.8 K 2024-11-08T02:30:52,286 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4e3aaadbabdf44709db7385dae253f65, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731033047785 2024-11-08T02:30:52,286 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 842592239c084abdb1dfb9f614741d27, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731033047785 2024-11-08T02:30:52,286 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 55b52b8bcc004b7297f591dee5a14479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731033048953 2024-11-08T02:30:52,286 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8e8e83eae1504899b7a9f40836593bef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731033048953 2024-11-08T02:30:52,287 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c68484d5f2e043119c541bab73579167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731033050092 2024-11-08T02:30:52,287 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting a66ca2f19e7649c4a1e03ad69f413f16, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731033050089 2024-11-08T02:30:52,296 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:52,296 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:52,296 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/aaf527ed30da46d1ae8cbb7111bcbbeb is 50, key is test_row_0/A:col10/1731033051018/Put/seqid=0 2024-11-08T02:30:52,297 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/7feb41782d4046b3b23dc0454f4658ce is 50, key is test_row_0/B:col10/1731033051018/Put/seqid=0 2024-11-08T02:30:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741917_1093 (size=13187) 2024-11-08T02:30:52,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741918_1094 (size=13187) 2024-11-08T02:30:52,362 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:52,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-08T02:30:52,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,363 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:30:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:52,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/d5378faec0ee4f2cba95aeccd2d1a46c is 50, key is test_row_0/A:col10/1731033051043/Put/seqid=0 2024-11-08T02:30:52,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741919_1095 (size=12301) 2024-11-08T02:30:52,714 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/7feb41782d4046b3b23dc0454f4658ce as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7feb41782d4046b3b23dc0454f4658ce 2024-11-08T02:30:52,714 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/aaf527ed30da46d1ae8cbb7111bcbbeb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/aaf527ed30da46d1ae8cbb7111bcbbeb 2024-11-08T02:30:52,722 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into 7feb41782d4046b3b23dc0454f4658ce(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:52,722 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:52,722 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into aaf527ed30da46d1ae8cbb7111bcbbeb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:52,722 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033052284; duration=0sec 2024-11-08T02:30:52,722 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:52,722 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033052283; duration=0sec 2024-11-08T02:30:52,722 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:52,722 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:52,722 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:52,722 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:52,722 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:52,724 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:52,724 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:52,724 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:52,724 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/61812f9aa06344769879e77c1f86534f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/62b6d96c670a4670819bbe91dd31dd13, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/bd922b1622974f7eaff17baefa554d2a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.8 K 2024-11-08T02:30:52,724 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 61812f9aa06344769879e77c1f86534f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731033047785 2024-11-08T02:30:52,725 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 62b6d96c670a4670819bbe91dd31dd13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731033048953 2024-11-08T02:30:52,725 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting bd922b1622974f7eaff17baefa554d2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731033050092 2024-11-08T02:30:52,734 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:52,734 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/42d43edfeffa4e0993bc60795d5ffbf1 is 50, key is test_row_0/C:col10/1731033051018/Put/seqid=0 2024-11-08T02:30:52,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741920_1096 (size=13187) 2024-11-08T02:30:52,772 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/d5378faec0ee4f2cba95aeccd2d1a46c 2024-11-08T02:30:52,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/6413f707e237459b933a7d3bd196dfdd is 50, key is test_row_0/B:col10/1731033051043/Put/seqid=0 2024-11-08T02:30:52,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741921_1097 (size=12301) 2024-11-08T02:30:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-08T02:30:53,146 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/42d43edfeffa4e0993bc60795d5ffbf1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/42d43edfeffa4e0993bc60795d5ffbf1 2024-11-08T02:30:53,151 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 42d43edfeffa4e0993bc60795d5ffbf1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:53,151 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:53,151 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033052284; duration=0sec 2024-11-08T02:30:53,152 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:53,152 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:53,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:53,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. as already flushing 2024-11-08T02:30:53,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033113170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033113171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033113174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033113174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,186 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/6413f707e237459b933a7d3bd196dfdd 2024-11-08T02:30:53,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/076d383cba254b9bb8a7d38fe937062e is 50, key is test_row_0/C:col10/1731033051043/Put/seqid=0 2024-11-08T02:30:53,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741922_1098 (size=12301) 2024-11-08T02:30:53,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033113275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033113276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033113276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033113276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033113480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033113480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033113480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033113480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,602 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/076d383cba254b9bb8a7d38fe937062e 2024-11-08T02:30:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/d5378faec0ee4f2cba95aeccd2d1a46c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/d5378faec0ee4f2cba95aeccd2d1a46c 2024-11-08T02:30:53,617 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/d5378faec0ee4f2cba95aeccd2d1a46c, entries=150, sequenceid=387, filesize=12.0 K 2024-11-08T02:30:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/6413f707e237459b933a7d3bd196dfdd as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/6413f707e237459b933a7d3bd196dfdd 2024-11-08T02:30:53,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/6413f707e237459b933a7d3bd196dfdd, entries=150, sequenceid=387, filesize=12.0 K 2024-11-08T02:30:53,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/076d383cba254b9bb8a7d38fe937062e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/076d383cba254b9bb8a7d38fe937062e 2024-11-08T02:30:53,630 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/076d383cba254b9bb8a7d38fe937062e, entries=150, sequenceid=387, filesize=12.0 K 2024-11-08T02:30:53,631 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for dc8ad63d9a6a95258392be1a735a16e9 in 1269ms, sequenceid=387, compaction requested=false 2024-11-08T02:30:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=29 2024-11-08T02:30:53,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=29 2024-11-08T02:30:53,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T02:30:53,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6690 sec 2024-11-08T02:30:53,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees in 2.6730 sec 2024-11-08T02:30:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:53,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:30:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:53,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:53,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/f4494a903a7d4314a8bf57e021436376 is 50, key is test_row_0/A:col10/1731033053173/Put/seqid=0 2024-11-08T02:30:53,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741923_1099 (size=14741) 2024-11-08T02:30:53,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/f4494a903a7d4314a8bf57e021436376 2024-11-08T02:30:53,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033113803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033113804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033113804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033113805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e24fff4c069d4aab86e263202b75f6de is 50, key is test_row_0/B:col10/1731033053173/Put/seqid=0 2024-11-08T02:30:53,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741924_1100 (size=12301) 2024-11-08T02:30:53,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e24fff4c069d4aab86e263202b75f6de 2024-11-08T02:30:53,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6d84a36c10254e6fb805e8dc1143cf62 is 50, key is test_row_0/C:col10/1731033053173/Put/seqid=0 2024-11-08T02:30:53,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741925_1101 (size=12301) 2024-11-08T02:30:53,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6d84a36c10254e6fb805e8dc1143cf62 2024-11-08T02:30:53,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/f4494a903a7d4314a8bf57e021436376 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f4494a903a7d4314a8bf57e021436376 2024-11-08T02:30:53,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f4494a903a7d4314a8bf57e021436376, entries=200, sequenceid=407, filesize=14.4 K 2024-11-08T02:30:53,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/e24fff4c069d4aab86e263202b75f6de as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e24fff4c069d4aab86e263202b75f6de 2024-11-08T02:30:53,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e24fff4c069d4aab86e263202b75f6de, entries=150, sequenceid=407, filesize=12.0 K 2024-11-08T02:30:53,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/6d84a36c10254e6fb805e8dc1143cf62 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6d84a36c10254e6fb805e8dc1143cf62 2024-11-08T02:30:53,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6d84a36c10254e6fb805e8dc1143cf62, entries=150, sequenceid=407, filesize=12.0 K 2024-11-08T02:30:53,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for dc8ad63d9a6a95258392be1a735a16e9 in 91ms, sequenceid=407, compaction requested=true 2024-11-08T02:30:53,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:53,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:53,875 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:53,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:53,875 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:53,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:53,877 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:53,877 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:53,877 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:53,877 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7feb41782d4046b3b23dc0454f4658ce, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/6413f707e237459b933a7d3bd196dfdd, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e24fff4c069d4aab86e263202b75f6de] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.9 K 2024-11-08T02:30:53,877 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:53,877 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:53,878 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:53,878 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/aaf527ed30da46d1ae8cbb7111bcbbeb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/d5378faec0ee4f2cba95aeccd2d1a46c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f4494a903a7d4314a8bf57e021436376] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=39.3 K 2024-11-08T02:30:53,878 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting aaf527ed30da46d1ae8cbb7111bcbbeb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731033050092 2024-11-08T02:30:53,878 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 7feb41782d4046b3b23dc0454f4658ce, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731033050092 2024-11-08T02:30:53,879 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6413f707e237459b933a7d3bd196dfdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731033051039 2024-11-08T02:30:53,879 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d5378faec0ee4f2cba95aeccd2d1a46c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731033051039 2024-11-08T02:30:53,879 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e24fff4c069d4aab86e263202b75f6de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731033053173 2024-11-08T02:30:53,879 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f4494a903a7d4314a8bf57e021436376, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731033053173 2024-11-08T02:30:53,891 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:53,891 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:53,891 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/f989d5f496544d1fa3a2f471b37e16c7 is 50, key is test_row_0/B:col10/1731033053173/Put/seqid=0 2024-11-08T02:30:53,891 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/c6b6e58385794100ada7a2521e65be1e is 50, key is test_row_0/A:col10/1731033053173/Put/seqid=0 2024-11-08T02:30:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741927_1103 (size=13289) 2024-11-08T02:30:53,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741926_1102 (size=13289) 2024-11-08T02:30:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:53,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:30:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:53,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/cf5d612eafa74bc1a3aa702d34b62309 is 50, key is test_row_0/A:col10/1731033053804/Put/seqid=0 2024-11-08T02:30:53,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741928_1104 (size=14741) 2024-11-08T02:30:53,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/cf5d612eafa74bc1a3aa702d34b62309 2024-11-08T02:30:53,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033113923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033113924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033113925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033113925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:53,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/931303a3a38d47cd9f2d8da8389393a7 is 50, key is test_row_0/B:col10/1731033053804/Put/seqid=0 2024-11-08T02:30:53,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741929_1105 (size=12301) 2024-11-08T02:30:53,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/931303a3a38d47cd9f2d8da8389393a7 2024-11-08T02:30:53,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/efbe8241063a427d9604bc2120f93ec6 is 50, key is test_row_0/C:col10/1731033053804/Put/seqid=0 2024-11-08T02:30:53,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741930_1106 (size=12301) 2024-11-08T02:30:54,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033114027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033114027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033114029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033114030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42894 deadline: 1731033114213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,215 DEBUG [Thread-151 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4122 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:30:54,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033114231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033114232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033114232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033114240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,307 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/c6b6e58385794100ada7a2521e65be1e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/c6b6e58385794100ada7a2521e65be1e 2024-11-08T02:30:54,310 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/f989d5f496544d1fa3a2f471b37e16c7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f989d5f496544d1fa3a2f471b37e16c7 2024-11-08T02:30:54,314 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into c6b6e58385794100ada7a2521e65be1e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:54,314 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:54,314 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033053875; duration=0sec 2024-11-08T02:30:54,314 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:54,314 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:54,314 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:54,315 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:54,315 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:54,316 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:54,316 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/42d43edfeffa4e0993bc60795d5ffbf1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/076d383cba254b9bb8a7d38fe937062e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6d84a36c10254e6fb805e8dc1143cf62] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=36.9 K 2024-11-08T02:30:54,316 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 42d43edfeffa4e0993bc60795d5ffbf1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731033050092 2024-11-08T02:30:54,317 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into f989d5f496544d1fa3a2f471b37e16c7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:54,317 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:54,317 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033053875; duration=0sec 2024-11-08T02:30:54,317 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 076d383cba254b9bb8a7d38fe937062e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731033051039 2024-11-08T02:30:54,317 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:54,317 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:54,317 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6d84a36c10254e6fb805e8dc1143cf62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731033053173 2024-11-08T02:30:54,324 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:54,325 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/3cd38e8ab03b481e85aed8261ac1b8ef is 50, key is test_row_0/C:col10/1731033053173/Put/seqid=0 2024-11-08T02:30:54,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741931_1107 (size=13289) 2024-11-08T02:30:54,340 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/3cd38e8ab03b481e85aed8261ac1b8ef as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3cd38e8ab03b481e85aed8261ac1b8ef 2024-11-08T02:30:54,350 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 3cd38e8ab03b481e85aed8261ac1b8ef(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:54,350 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:54,350 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033053876; duration=0sec 2024-11-08T02:30:54,350 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:54,350 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:54,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/efbe8241063a427d9604bc2120f93ec6 2024-11-08T02:30:54,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/cf5d612eafa74bc1a3aa702d34b62309 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/cf5d612eafa74bc1a3aa702d34b62309 2024-11-08T02:30:54,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/cf5d612eafa74bc1a3aa702d34b62309, entries=200, sequenceid=427, filesize=14.4 K 2024-11-08T02:30:54,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/931303a3a38d47cd9f2d8da8389393a7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/931303a3a38d47cd9f2d8da8389393a7 2024-11-08T02:30:54,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/931303a3a38d47cd9f2d8da8389393a7, entries=150, sequenceid=427, filesize=12.0 K 2024-11-08T02:30:54,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/efbe8241063a427d9604bc2120f93ec6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/efbe8241063a427d9604bc2120f93ec6 2024-11-08T02:30:54,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/efbe8241063a427d9604bc2120f93ec6, entries=150, sequenceid=427, filesize=12.0 K 2024-11-08T02:30:54,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for dc8ad63d9a6a95258392be1a735a16e9 in 476ms, sequenceid=427, compaction requested=false 2024-11-08T02:30:54,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:54,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:54,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:30:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:54,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:54,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/862e2ba41cb048929835f5eb25830fc0 is 50, key is test_row_0/A:col10/1731033053924/Put/seqid=0 2024-11-08T02:30:54,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741932_1108 (size=14741) 2024-11-08T02:30:54,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/862e2ba41cb048929835f5eb25830fc0 2024-11-08T02:30:54,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/92f1fa722a37473b91638da87e4d7374 is 50, key is test_row_0/B:col10/1731033053924/Put/seqid=0 2024-11-08T02:30:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033114549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033114550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033114554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033114556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741933_1109 (size=12301) 2024-11-08T02:30:54,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033114655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033114655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033114657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033114658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033114858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033114858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033114861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:54,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033114861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:54,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/92f1fa722a37473b91638da87e4d7374 2024-11-08T02:30:54,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/f116cc8804d847168596d670112830af is 50, key is test_row_0/C:col10/1731033053924/Put/seqid=0 2024-11-08T02:30:54,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741934_1110 (size=12301) 2024-11-08T02:30:55,048 DEBUG [Thread-164 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d2a8e08 to 127.0.0.1:56877 2024-11-08T02:30:55,048 DEBUG [Thread-164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,048 DEBUG [Thread-162 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x70267494 to 127.0.0.1:56877 2024-11-08T02:30:55,048 DEBUG [Thread-162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,048 DEBUG [Thread-160 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x295cb1ac to 127.0.0.1:56877 2024-11-08T02:30:55,049 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,049 DEBUG [Thread-166 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2c915d17 to 127.0.0.1:56877 2024-11-08T02:30:55,049 DEBUG [Thread-166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,056 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:30:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-08T02:30:55,067 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 28 completed 2024-11-08T02:30:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42884 deadline: 1731033115162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731033115162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:55,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42892 deadline: 1731033115164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:55,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:30:55,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42866 deadline: 1731033115164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:55,233 ERROR [LeaseRenewer:jenkins@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46319,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:55,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/f116cc8804d847168596d670112830af 2024-11-08T02:30:55,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/862e2ba41cb048929835f5eb25830fc0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/862e2ba41cb048929835f5eb25830fc0 2024-11-08T02:30:55,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/862e2ba41cb048929835f5eb25830fc0, entries=200, sequenceid=448, filesize=14.4 K 2024-11-08T02:30:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/92f1fa722a37473b91638da87e4d7374 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/92f1fa722a37473b91638da87e4d7374 2024-11-08T02:30:55,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/92f1fa722a37473b91638da87e4d7374, entries=150, sequenceid=448, filesize=12.0 K 2024-11-08T02:30:55,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/f116cc8804d847168596d670112830af as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f116cc8804d847168596d670112830af 2024-11-08T02:30:55,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f116cc8804d847168596d670112830af, entries=150, sequenceid=448, filesize=12.0 K 2024-11-08T02:30:55,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for dc8ad63d9a6a95258392be1a735a16e9 in 875ms, sequenceid=448, compaction requested=true 2024-11-08T02:30:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:55,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:30:55,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:55,411 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:55,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:30:55,411 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:55,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:55,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc8ad63d9a6a95258392be1a735a16e9:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:30:55,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:55,412 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:55,412 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:55,412 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/A is initiating minor compaction (all files) 2024-11-08T02:30:55,412 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/B is initiating minor compaction (all files) 2024-11-08T02:30:55,412 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/A in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:55,412 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/B in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:55,412 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/c6b6e58385794100ada7a2521e65be1e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/cf5d612eafa74bc1a3aa702d34b62309, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/862e2ba41cb048929835f5eb25830fc0] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=41.8 K 2024-11-08T02:30:55,412 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f989d5f496544d1fa3a2f471b37e16c7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/931303a3a38d47cd9f2d8da8389393a7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/92f1fa722a37473b91638da87e4d7374] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=37.0 K 2024-11-08T02:30:55,412 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f989d5f496544d1fa3a2f471b37e16c7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731033053173 2024-11-08T02:30:55,412 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting c6b6e58385794100ada7a2521e65be1e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731033053173 2024-11-08T02:30:55,413 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 931303a3a38d47cd9f2d8da8389393a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731033053803 2024-11-08T02:30:55,413 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting cf5d612eafa74bc1a3aa702d34b62309, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731033053801 2024-11-08T02:30:55,413 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 92f1fa722a37473b91638da87e4d7374, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731033053918 2024-11-08T02:30:55,413 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 862e2ba41cb048929835f5eb25830fc0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731033053918 2024-11-08T02:30:55,421 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#A#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:55,421 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#B#compaction#97 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:55,421 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b122dcb67ef44134a3faebeb0a7475e3 is 50, key is test_row_0/A:col10/1731033053924/Put/seqid=0 2024-11-08T02:30:55,421 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/3eee82f00e8948e992f9b72229b35751 is 50, key is test_row_0/B:col10/1731033053924/Put/seqid=0 2024-11-08T02:30:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741936_1112 (size=13391) 2024-11-08T02:30:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741935_1111 (size=13391) 2024-11-08T02:30:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:55,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:30:55,669 DEBUG [Thread-157 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38766d64 to 127.0.0.1:56877 2024-11-08T02:30:55,669 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:55,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:55,671 DEBUG [Thread-149 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f343a4d to 127.0.0.1:56877 2024-11-08T02:30:55,671 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,672 DEBUG [Thread-155 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5400112e to 127.0.0.1:56877 2024-11-08T02:30:55,672 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,672 DEBUG [Thread-153 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x478bae6b to 127.0.0.1:56877 2024-11-08T02:30:55,673 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:55,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/0a254ea50c2b44069bb6540fe4646db3 is 50, key is test_row_0/A:col10/1731033054547/Put/seqid=0 2024-11-08T02:30:55,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741937_1113 (size=12301) 2024-11-08T02:30:55,836 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b122dcb67ef44134a3faebeb0a7475e3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b122dcb67ef44134a3faebeb0a7475e3 2024-11-08T02:30:55,836 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/3eee82f00e8948e992f9b72229b35751 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/3eee82f00e8948e992f9b72229b35751 2024-11-08T02:30:55,842 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/B of dc8ad63d9a6a95258392be1a735a16e9 into 3eee82f00e8948e992f9b72229b35751(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:55,842 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/A of dc8ad63d9a6a95258392be1a735a16e9 into b122dcb67ef44134a3faebeb0a7475e3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:55,842 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:55,842 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:55,842 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/B, priority=13, startTime=1731033055411; duration=0sec 2024-11-08T02:30:55,842 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/A, priority=13, startTime=1731033055410; duration=0sec 2024-11-08T02:30:55,842 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:55,842 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:30:55,842 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:B 2024-11-08T02:30:55,842 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:A 2024-11-08T02:30:55,842 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:30:55,844 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:30:55,844 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): dc8ad63d9a6a95258392be1a735a16e9/C is initiating minor compaction (all files) 2024-11-08T02:30:55,844 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of dc8ad63d9a6a95258392be1a735a16e9/C in TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:55,844 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3cd38e8ab03b481e85aed8261ac1b8ef, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/efbe8241063a427d9604bc2120f93ec6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f116cc8804d847168596d670112830af] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp, totalSize=37.0 K 2024-11-08T02:30:55,845 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 3cd38e8ab03b481e85aed8261ac1b8ef, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731033053173 2024-11-08T02:30:55,845 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting efbe8241063a427d9604bc2120f93ec6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731033053803 2024-11-08T02:30:55,845 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f116cc8804d847168596d670112830af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731033053918 2024-11-08T02:30:55,853 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc8ad63d9a6a95258392be1a735a16e9#C#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:30:55,854 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/1d1eebaf90f24e80a0d169df881e8a26 is 50, key is test_row_0/C:col10/1731033053924/Put/seqid=0 2024-11-08T02:30:55,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741938_1114 (size=13391) 2024-11-08T02:30:56,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/0a254ea50c2b44069bb6540fe4646db3 2024-11-08T02:30:56,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/32786562c1194d67b7e2cfd7433a9439 is 50, key is test_row_0/B:col10/1731033054547/Put/seqid=0 2024-11-08T02:30:56,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741939_1115 (size=12301) 2024-11-08T02:30:56,271 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/1d1eebaf90f24e80a0d169df881e8a26 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1d1eebaf90f24e80a0d169df881e8a26 2024-11-08T02:30:56,275 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in dc8ad63d9a6a95258392be1a735a16e9/C of dc8ad63d9a6a95258392be1a735a16e9 into 1d1eebaf90f24e80a0d169df881e8a26(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:30:56,275 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:56,275 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9., storeName=dc8ad63d9a6a95258392be1a735a16e9/C, priority=13, startTime=1731033055411; duration=0sec 2024-11-08T02:30:56,275 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:30:56,276 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc8ad63d9a6a95258392be1a735a16e9:C 2024-11-08T02:30:56,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/32786562c1194d67b7e2cfd7433a9439 2024-11-08T02:30:56,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/59f49fd6b62b4aa1967dc843f1384049 is 50, key is test_row_0/C:col10/1731033054547/Put/seqid=0 2024-11-08T02:30:56,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741940_1116 (size=12301) 2024-11-08T02:30:56,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/59f49fd6b62b4aa1967dc843f1384049 2024-11-08T02:30:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/0a254ea50c2b44069bb6540fe4646db3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0a254ea50c2b44069bb6540fe4646db3 2024-11-08T02:30:56,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0a254ea50c2b44069bb6540fe4646db3, entries=150, sequenceid=469, filesize=12.0 K 2024-11-08T02:30:56,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/32786562c1194d67b7e2cfd7433a9439 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/32786562c1194d67b7e2cfd7433a9439 2024-11-08T02:30:56,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/32786562c1194d67b7e2cfd7433a9439, entries=150, sequenceid=469, filesize=12.0 K 2024-11-08T02:30:56,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/59f49fd6b62b4aa1967dc843f1384049 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/59f49fd6b62b4aa1967dc843f1384049 2024-11-08T02:30:56,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/59f49fd6b62b4aa1967dc843f1384049, entries=150, sequenceid=469, filesize=12.0 K 2024-11-08T02:30:56,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=13.42 KB/13740 for dc8ad63d9a6a95258392be1a735a16e9 in 1306ms, sequenceid=469, compaction requested=false 2024-11-08T02:30:56,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:58,248 DEBUG [Thread-151 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22cb07dd to 127.0.0.1:56877 2024-11-08T02:30:58,248 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:58,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:30:58,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 129 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6463 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6394 2024-11-08T02:30:58,250 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:30:58,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2754 2024-11-08T02:30:58,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 8259 rows 2024-11-08T02:30:58,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2750 2024-11-08T02:30:58,251 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 8247 rows 2024-11-08T02:30:58,251 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T02:30:58,251 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f6e36fe to 127.0.0.1:56877 2024-11-08T02:30:58,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:30:58,256 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-08T02:30:58,260 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:30:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:30:58,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033058266"}]},"ts":"1731033058266"} 2024-11-08T02:30:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T02:30:58,267 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:30:58,311 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:30:58,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:30:58,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc8ad63d9a6a95258392be1a735a16e9, UNASSIGN}] 2024-11-08T02:30:58,323 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc8ad63d9a6a95258392be1a735a16e9, UNASSIGN 2024-11-08T02:30:58,324 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=dc8ad63d9a6a95258392be1a735a16e9, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:30:58,325 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:30:58,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; CloseRegionProcedure dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:30:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T02:30:58,480 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:30:58,481 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] handler.UnassignRegionHandler(124): Close dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:58,481 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(1681): Closing dc8ad63d9a6a95258392be1a735a16e9, disabling compactions & flushes 2024-11-08T02:30:58,482 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. after waiting 0 ms 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:58,482 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(2837): Flushing dc8ad63d9a6a95258392be1a735a16e9 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=A 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=B 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc8ad63d9a6a95258392be1a735a16e9, store=C 2024-11-08T02:30:58,482 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:30:58,487 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b1da00d206664cd3a6e75a57aa6167f5 is 50, key is test_row_0/A:col10/1731033055670/Put/seqid=0 2024-11-08T02:30:58,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741941_1117 (size=9857) 2024-11-08T02:30:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T02:30:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T02:30:58,893 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b1da00d206664cd3a6e75a57aa6167f5 2024-11-08T02:30:58,905 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/391071836ddb4bcbb8063e7ed377ca1e is 50, key is test_row_0/B:col10/1731033055670/Put/seqid=0 2024-11-08T02:30:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741942_1118 (size=9857) 2024-11-08T02:30:59,312 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/391071836ddb4bcbb8063e7ed377ca1e 2024-11-08T02:30:59,327 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/33c610e76c0340438f68e69f2bfcc7eb is 50, key is test_row_0/C:col10/1731033055670/Put/seqid=0 2024-11-08T02:30:59,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741943_1119 (size=9857) 2024-11-08T02:30:59,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T02:30:59,733 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/33c610e76c0340438f68e69f2bfcc7eb 2024-11-08T02:30:59,739 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/A/b1da00d206664cd3a6e75a57aa6167f5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b1da00d206664cd3a6e75a57aa6167f5 2024-11-08T02:30:59,745 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b1da00d206664cd3a6e75a57aa6167f5, entries=100, sequenceid=478, filesize=9.6 K 2024-11-08T02:30:59,746 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/B/391071836ddb4bcbb8063e7ed377ca1e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/391071836ddb4bcbb8063e7ed377ca1e 2024-11-08T02:30:59,751 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/391071836ddb4bcbb8063e7ed377ca1e, entries=100, sequenceid=478, filesize=9.6 K 2024-11-08T02:30:59,752 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/.tmp/C/33c610e76c0340438f68e69f2bfcc7eb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/33c610e76c0340438f68e69f2bfcc7eb 2024-11-08T02:30:59,757 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/33c610e76c0340438f68e69f2bfcc7eb, entries=100, sequenceid=478, filesize=9.6 K 2024-11-08T02:30:59,758 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(3040): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for dc8ad63d9a6a95258392be1a735a16e9 in 1276ms, sequenceid=478, compaction requested=true 2024-11-08T02:30:59,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e51aa4859354965ae3aa4ea16fe5d6a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bf91c28a7e7643ad843808d15b26d8b9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e87cf96fc525412dac1435f5dad1b2eb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8ab7ec653ba24612a43f2cc86f5429e3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bb05790f2f1043a5b07a7d7a7a276d67, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0f64faca361c42e18ffca4dee99a7da5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f784536f74524ae8a1b418a70a890dc2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e572085a331e4ccab0498acb1328af5a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/1e132f0f974c4f9590fe439a676cb93d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/69e688857a1441cfb9d62bb0aae5e2ee, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/6884da23e7644bc8846904507bec8c5f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/14c696e97c264803aaee2c9bb2c3de64, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/84a13b389fc342a68fce3d758f90cedf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/21a1ecd7aef243abb711cb683c7eb513, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/7e88852277354ac89b3e5c9b67ce56c7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/3cde3978e2524cd0b09ebc985c725fe5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/ef3b096ef602409a93dcac074b6ba8bf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b6ed57a064d8496991d23ce9ce0f5dc1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8a6758b0674d4ccd869d185811bd248b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8232298bf6e14c5aaa9375c956cbc828, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/45e30d16a45a426db7fcb6245b01773a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/700f9c6328754a038287308fc3427625, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/74a36d08f5384a6e91ef4622bc58efe5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/4e3aaadbabdf44709db7385dae253f65, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e8e83eae1504899b7a9f40836593bef, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/a66ca2f19e7649c4a1e03ad69f413f16, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/aaf527ed30da46d1ae8cbb7111bcbbeb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/d5378faec0ee4f2cba95aeccd2d1a46c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f4494a903a7d4314a8bf57e021436376, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/c6b6e58385794100ada7a2521e65be1e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/cf5d612eafa74bc1a3aa702d34b62309, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/862e2ba41cb048929835f5eb25830fc0] to archive 2024-11-08T02:30:59,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:30:59,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e51aa4859354965ae3aa4ea16fe5d6a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e51aa4859354965ae3aa4ea16fe5d6a 2024-11-08T02:30:59,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bf91c28a7e7643ad843808d15b26d8b9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bf91c28a7e7643ad843808d15b26d8b9 2024-11-08T02:30:59,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e87cf96fc525412dac1435f5dad1b2eb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e87cf96fc525412dac1435f5dad1b2eb 2024-11-08T02:30:59,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8ab7ec653ba24612a43f2cc86f5429e3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8ab7ec653ba24612a43f2cc86f5429e3 2024-11-08T02:30:59,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bb05790f2f1043a5b07a7d7a7a276d67 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/bb05790f2f1043a5b07a7d7a7a276d67 2024-11-08T02:30:59,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0f64faca361c42e18ffca4dee99a7da5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0f64faca361c42e18ffca4dee99a7da5 2024-11-08T02:30:59,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f784536f74524ae8a1b418a70a890dc2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f784536f74524ae8a1b418a70a890dc2 2024-11-08T02:30:59,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e572085a331e4ccab0498acb1328af5a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/e572085a331e4ccab0498acb1328af5a 2024-11-08T02:30:59,778 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46319,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:30:59,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/1e132f0f974c4f9590fe439a676cb93d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/1e132f0f974c4f9590fe439a676cb93d 2024-11-08T02:30:59,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/69e688857a1441cfb9d62bb0aae5e2ee to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/69e688857a1441cfb9d62bb0aae5e2ee 2024-11-08T02:30:59,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/6884da23e7644bc8846904507bec8c5f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/6884da23e7644bc8846904507bec8c5f 2024-11-08T02:30:59,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/14c696e97c264803aaee2c9bb2c3de64 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/14c696e97c264803aaee2c9bb2c3de64 2024-11-08T02:30:59,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/84a13b389fc342a68fce3d758f90cedf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/84a13b389fc342a68fce3d758f90cedf 2024-11-08T02:30:59,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/21a1ecd7aef243abb711cb683c7eb513 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/21a1ecd7aef243abb711cb683c7eb513 2024-11-08T02:30:59,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/7e88852277354ac89b3e5c9b67ce56c7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/7e88852277354ac89b3e5c9b67ce56c7 2024-11-08T02:30:59,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/3cde3978e2524cd0b09ebc985c725fe5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/3cde3978e2524cd0b09ebc985c725fe5 2024-11-08T02:30:59,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/ef3b096ef602409a93dcac074b6ba8bf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/ef3b096ef602409a93dcac074b6ba8bf 2024-11-08T02:30:59,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b6ed57a064d8496991d23ce9ce0f5dc1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b6ed57a064d8496991d23ce9ce0f5dc1 2024-11-08T02:30:59,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8a6758b0674d4ccd869d185811bd248b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8a6758b0674d4ccd869d185811bd248b 2024-11-08T02:30:59,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8232298bf6e14c5aaa9375c956cbc828 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8232298bf6e14c5aaa9375c956cbc828 2024-11-08T02:30:59,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/45e30d16a45a426db7fcb6245b01773a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/45e30d16a45a426db7fcb6245b01773a 2024-11-08T02:30:59,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/700f9c6328754a038287308fc3427625 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/700f9c6328754a038287308fc3427625 2024-11-08T02:30:59,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/74a36d08f5384a6e91ef4622bc58efe5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/74a36d08f5384a6e91ef4622bc58efe5 2024-11-08T02:30:59,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/4e3aaadbabdf44709db7385dae253f65 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/4e3aaadbabdf44709db7385dae253f65 2024-11-08T02:30:59,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e8e83eae1504899b7a9f40836593bef to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/8e8e83eae1504899b7a9f40836593bef 2024-11-08T02:30:59,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/a66ca2f19e7649c4a1e03ad69f413f16 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/a66ca2f19e7649c4a1e03ad69f413f16 2024-11-08T02:30:59,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/aaf527ed30da46d1ae8cbb7111bcbbeb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/aaf527ed30da46d1ae8cbb7111bcbbeb 2024-11-08T02:30:59,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/d5378faec0ee4f2cba95aeccd2d1a46c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/d5378faec0ee4f2cba95aeccd2d1a46c 2024-11-08T02:30:59,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f4494a903a7d4314a8bf57e021436376 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/f4494a903a7d4314a8bf57e021436376 2024-11-08T02:30:59,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/c6b6e58385794100ada7a2521e65be1e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/c6b6e58385794100ada7a2521e65be1e 2024-11-08T02:30:59,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/cf5d612eafa74bc1a3aa702d34b62309 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/cf5d612eafa74bc1a3aa702d34b62309 2024-11-08T02:30:59,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/862e2ba41cb048929835f5eb25830fc0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/862e2ba41cb048929835f5eb25830fc0 2024-11-08T02:30:59,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/087a079dc85448dfa456ce6acefc97f1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c5c9a58bec5a4ef0801ee4eb89e12dd7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/90375c3fa5e04beab6ee277c69afde4f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d4c3641e887e4116805ce2e601d66f12, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55f40b39e06149d5a8597881d3bb5e15, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/74bc7a9971d740a4a293936f19fb5ffe, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7c63a5dbdf1f4b32b26bd1faafbaf9fe, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f14de3ee0d3e4505af15679444fdec89, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/060a7fb2ad2346b1bb7e0bac7750df24, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fea1c374518d472fa6470e99d9518d83, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e36806a0912b4ac38ce6a5024abacdf4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e236aaa4806c4a01b05c1bfff3dfcf03, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/77eeb19648124bfe880e378ad48e23f7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e54d0f37a8d5480b8315d48b1566166d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fc60bda880554edf92004fd4a6520db2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/041e4001806f4275817051c7d13c2728, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/dec1c0b4e9324022a27677f9a4ed04b8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/97a3d87bf8fe492ab15ff3f107c2c7b8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/2c614466f2c14138ab4d0827038b3842, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fcf0d486e6d14735841e61685c87ed34, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/70bd1abe1b504eb6a46da755b9cd2146, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d425c40746e248c4b1fdb1848a98267e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/842592239c084abdb1dfb9f614741d27, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/740fa2c3a7b347da9d5131703ba19f87, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55b52b8bcc004b7297f591dee5a14479, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7feb41782d4046b3b23dc0454f4658ce, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c68484d5f2e043119c541bab73579167, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/6413f707e237459b933a7d3bd196dfdd, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f989d5f496544d1fa3a2f471b37e16c7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e24fff4c069d4aab86e263202b75f6de, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/931303a3a38d47cd9f2d8da8389393a7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/92f1fa722a37473b91638da87e4d7374] to archive 2024-11-08T02:30:59,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:30:59,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/087a079dc85448dfa456ce6acefc97f1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/087a079dc85448dfa456ce6acefc97f1 2024-11-08T02:30:59,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c5c9a58bec5a4ef0801ee4eb89e12dd7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c5c9a58bec5a4ef0801ee4eb89e12dd7 2024-11-08T02:30:59,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/90375c3fa5e04beab6ee277c69afde4f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/90375c3fa5e04beab6ee277c69afde4f 2024-11-08T02:30:59,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d4c3641e887e4116805ce2e601d66f12 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d4c3641e887e4116805ce2e601d66f12 2024-11-08T02:30:59,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55f40b39e06149d5a8597881d3bb5e15 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55f40b39e06149d5a8597881d3bb5e15 2024-11-08T02:30:59,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/74bc7a9971d740a4a293936f19fb5ffe to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/74bc7a9971d740a4a293936f19fb5ffe 2024-11-08T02:30:59,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7c63a5dbdf1f4b32b26bd1faafbaf9fe to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7c63a5dbdf1f4b32b26bd1faafbaf9fe 2024-11-08T02:30:59,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f14de3ee0d3e4505af15679444fdec89 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f14de3ee0d3e4505af15679444fdec89 2024-11-08T02:30:59,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/060a7fb2ad2346b1bb7e0bac7750df24 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/060a7fb2ad2346b1bb7e0bac7750df24 2024-11-08T02:30:59,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fea1c374518d472fa6470e99d9518d83 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fea1c374518d472fa6470e99d9518d83 2024-11-08T02:30:59,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e36806a0912b4ac38ce6a5024abacdf4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e36806a0912b4ac38ce6a5024abacdf4 2024-11-08T02:30:59,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e236aaa4806c4a01b05c1bfff3dfcf03 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e236aaa4806c4a01b05c1bfff3dfcf03 2024-11-08T02:30:59,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/77eeb19648124bfe880e378ad48e23f7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/77eeb19648124bfe880e378ad48e23f7 2024-11-08T02:30:59,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e54d0f37a8d5480b8315d48b1566166d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e54d0f37a8d5480b8315d48b1566166d 2024-11-08T02:30:59,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fc60bda880554edf92004fd4a6520db2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fc60bda880554edf92004fd4a6520db2 2024-11-08T02:30:59,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/041e4001806f4275817051c7d13c2728 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/041e4001806f4275817051c7d13c2728 2024-11-08T02:30:59,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/dec1c0b4e9324022a27677f9a4ed04b8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/dec1c0b4e9324022a27677f9a4ed04b8 2024-11-08T02:30:59,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/97a3d87bf8fe492ab15ff3f107c2c7b8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/97a3d87bf8fe492ab15ff3f107c2c7b8 2024-11-08T02:30:59,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/2c614466f2c14138ab4d0827038b3842 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/2c614466f2c14138ab4d0827038b3842 2024-11-08T02:30:59,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fcf0d486e6d14735841e61685c87ed34 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/fcf0d486e6d14735841e61685c87ed34 2024-11-08T02:30:59,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/70bd1abe1b504eb6a46da755b9cd2146 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/70bd1abe1b504eb6a46da755b9cd2146 2024-11-08T02:30:59,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d425c40746e248c4b1fdb1848a98267e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/d425c40746e248c4b1fdb1848a98267e 2024-11-08T02:30:59,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/842592239c084abdb1dfb9f614741d27 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/842592239c084abdb1dfb9f614741d27 2024-11-08T02:30:59,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/740fa2c3a7b347da9d5131703ba19f87 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/740fa2c3a7b347da9d5131703ba19f87 2024-11-08T02:30:59,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55b52b8bcc004b7297f591dee5a14479 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/55b52b8bcc004b7297f591dee5a14479 2024-11-08T02:30:59,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7feb41782d4046b3b23dc0454f4658ce to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/7feb41782d4046b3b23dc0454f4658ce 2024-11-08T02:30:59,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c68484d5f2e043119c541bab73579167 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/c68484d5f2e043119c541bab73579167 2024-11-08T02:30:59,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/6413f707e237459b933a7d3bd196dfdd to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/6413f707e237459b933a7d3bd196dfdd 2024-11-08T02:30:59,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f989d5f496544d1fa3a2f471b37e16c7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/f989d5f496544d1fa3a2f471b37e16c7 2024-11-08T02:30:59,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e24fff4c069d4aab86e263202b75f6de to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/e24fff4c069d4aab86e263202b75f6de 2024-11-08T02:30:59,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/931303a3a38d47cd9f2d8da8389393a7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/931303a3a38d47cd9f2d8da8389393a7 2024-11-08T02:30:59,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/92f1fa722a37473b91638da87e4d7374 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/92f1fa722a37473b91638da87e4d7374 2024-11-08T02:30:59,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/7facc17547e9424eb7e99849ed8e6081, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3e7a9d374e79465bb013ea80a6159048, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c3622dab435b401fbe3b6f3e8d9f37d0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e028be4d907048afadf39b845aabcc09, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/945bc43e0fc942718e5a479fd9f533ae, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/d53fc06a86694b1d915e0fd8bf3e87c9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1c544aaeefcc4d94b658030f624691fb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6078050be37d4e6cb00883789f09a864, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/955778f4ac624705970d028afa4b2815, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/9d2dbb80f70a463ca0c35bb79a531059, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6515238286e149a4b10de0b868d28df0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/ff9647b63a564c5eb8caf6ce6455db93, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e7fbdb3fc1cb434992604dbce35888c9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/0d63cafa289940e7802693db3f520ec0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/94ebabbb2fd84be1b67ac1841c52cb10, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/8fe03ff735b64102a47cc0395662c18d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f46cbe4866c64fba926dd3eb6a4ff8fb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b3963743e23c42f3bf26d24f93469909, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/fac02c2c50924e8da643285bcd6f2c57, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/40fd639b5f14437b9e1cbb2bd98b9244, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/09a0f4a4be894aeba47a46da4bc5ce51, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/61812f9aa06344769879e77c1f86534f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c336c88034fc4fbaa9193522e453408b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/62b6d96c670a4670819bbe91dd31dd13, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/42d43edfeffa4e0993bc60795d5ffbf1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/bd922b1622974f7eaff17baefa554d2a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/076d383cba254b9bb8a7d38fe937062e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3cd38e8ab03b481e85aed8261ac1b8ef, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6d84a36c10254e6fb805e8dc1143cf62, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/efbe8241063a427d9604bc2120f93ec6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f116cc8804d847168596d670112830af] to archive 2024-11-08T02:30:59,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:30:59,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/7facc17547e9424eb7e99849ed8e6081 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/7facc17547e9424eb7e99849ed8e6081 2024-11-08T02:30:59,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3e7a9d374e79465bb013ea80a6159048 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3e7a9d374e79465bb013ea80a6159048 2024-11-08T02:30:59,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c3622dab435b401fbe3b6f3e8d9f37d0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c3622dab435b401fbe3b6f3e8d9f37d0 2024-11-08T02:30:59,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e028be4d907048afadf39b845aabcc09 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e028be4d907048afadf39b845aabcc09 2024-11-08T02:30:59,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/945bc43e0fc942718e5a479fd9f533ae to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/945bc43e0fc942718e5a479fd9f533ae 2024-11-08T02:30:59,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/d53fc06a86694b1d915e0fd8bf3e87c9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/d53fc06a86694b1d915e0fd8bf3e87c9 2024-11-08T02:30:59,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1c544aaeefcc4d94b658030f624691fb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1c544aaeefcc4d94b658030f624691fb 2024-11-08T02:30:59,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6078050be37d4e6cb00883789f09a864 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6078050be37d4e6cb00883789f09a864 2024-11-08T02:30:59,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/955778f4ac624705970d028afa4b2815 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/955778f4ac624705970d028afa4b2815 2024-11-08T02:30:59,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/9d2dbb80f70a463ca0c35bb79a531059 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/9d2dbb80f70a463ca0c35bb79a531059 2024-11-08T02:30:59,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6515238286e149a4b10de0b868d28df0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6515238286e149a4b10de0b868d28df0 2024-11-08T02:30:59,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/ff9647b63a564c5eb8caf6ce6455db93 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/ff9647b63a564c5eb8caf6ce6455db93 2024-11-08T02:30:59,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e7fbdb3fc1cb434992604dbce35888c9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/e7fbdb3fc1cb434992604dbce35888c9 2024-11-08T02:30:59,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/0d63cafa289940e7802693db3f520ec0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/0d63cafa289940e7802693db3f520ec0 2024-11-08T02:30:59,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/94ebabbb2fd84be1b67ac1841c52cb10 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/94ebabbb2fd84be1b67ac1841c52cb10 2024-11-08T02:30:59,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/8fe03ff735b64102a47cc0395662c18d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/8fe03ff735b64102a47cc0395662c18d 2024-11-08T02:30:59,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f46cbe4866c64fba926dd3eb6a4ff8fb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f46cbe4866c64fba926dd3eb6a4ff8fb 2024-11-08T02:30:59,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b3963743e23c42f3bf26d24f93469909 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b3963743e23c42f3bf26d24f93469909 2024-11-08T02:30:59,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/fac02c2c50924e8da643285bcd6f2c57 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/fac02c2c50924e8da643285bcd6f2c57 2024-11-08T02:30:59,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/40fd639b5f14437b9e1cbb2bd98b9244 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/40fd639b5f14437b9e1cbb2bd98b9244 2024-11-08T02:30:59,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/09a0f4a4be894aeba47a46da4bc5ce51 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/09a0f4a4be894aeba47a46da4bc5ce51 2024-11-08T02:30:59,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/b8af53f2f5f94f2bafb7d58f2d2c6f5c 2024-11-08T02:30:59,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/61812f9aa06344769879e77c1f86534f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/61812f9aa06344769879e77c1f86534f 2024-11-08T02:30:59,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c336c88034fc4fbaa9193522e453408b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/c336c88034fc4fbaa9193522e453408b 2024-11-08T02:30:59,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/62b6d96c670a4670819bbe91dd31dd13 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/62b6d96c670a4670819bbe91dd31dd13 2024-11-08T02:30:59,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/42d43edfeffa4e0993bc60795d5ffbf1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/42d43edfeffa4e0993bc60795d5ffbf1 2024-11-08T02:30:59,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/bd922b1622974f7eaff17baefa554d2a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/bd922b1622974f7eaff17baefa554d2a 2024-11-08T02:30:59,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/076d383cba254b9bb8a7d38fe937062e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/076d383cba254b9bb8a7d38fe937062e 2024-11-08T02:30:59,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3cd38e8ab03b481e85aed8261ac1b8ef to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/3cd38e8ab03b481e85aed8261ac1b8ef 2024-11-08T02:30:59,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6d84a36c10254e6fb805e8dc1143cf62 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/6d84a36c10254e6fb805e8dc1143cf62 2024-11-08T02:30:59,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/efbe8241063a427d9604bc2120f93ec6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/efbe8241063a427d9604bc2120f93ec6 2024-11-08T02:30:59,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f116cc8804d847168596d670112830af to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/f116cc8804d847168596d670112830af 2024-11-08T02:30:59,899 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/recovered.edits/481.seqid, newMaxSeqId=481, maxSeqId=1 2024-11-08T02:30:59,901 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9. 2024-11-08T02:30:59,901 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] regionserver.HRegion(1635): Region close journal for dc8ad63d9a6a95258392be1a735a16e9: 2024-11-08T02:30:59,903 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=33}] handler.UnassignRegionHandler(170): Closed dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:30:59,903 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=dc8ad63d9a6a95258392be1a735a16e9, regionState=CLOSED 2024-11-08T02:30:59,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=33, resume processing ppid=32 2024-11-08T02:30:59,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, ppid=32, state=SUCCESS; CloseRegionProcedure dc8ad63d9a6a95258392be1a735a16e9, server=ecdbff65922a,40577,1731033026611 in 1.5800 sec 2024-11-08T02:30:59,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-08T02:30:59,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc8ad63d9a6a95258392be1a735a16e9, UNASSIGN in 1.5830 sec 2024-11-08T02:30:59,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-08T02:30:59,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5960 sec 2024-11-08T02:30:59,912 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033059911"}]},"ts":"1731033059911"} 2024-11-08T02:30:59,913 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:30:59,953 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:30:59,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.6930 sec 2024-11-08T02:31:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-08T02:31:00,374 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 30 completed 2024-11-08T02:31:00,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:31:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:00,389 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=34, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:00,390 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=34, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-08T02:31:00,394 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:31:00,398 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/recovered.edits] 2024-11-08T02:31:00,401 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0a254ea50c2b44069bb6540fe4646db3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/0a254ea50c2b44069bb6540fe4646db3 2024-11-08T02:31:00,402 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b122dcb67ef44134a3faebeb0a7475e3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b122dcb67ef44134a3faebeb0a7475e3 2024-11-08T02:31:00,403 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b1da00d206664cd3a6e75a57aa6167f5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/A/b1da00d206664cd3a6e75a57aa6167f5 2024-11-08T02:31:00,406 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/32786562c1194d67b7e2cfd7433a9439 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/32786562c1194d67b7e2cfd7433a9439 2024-11-08T02:31:00,407 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/391071836ddb4bcbb8063e7ed377ca1e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/391071836ddb4bcbb8063e7ed377ca1e 2024-11-08T02:31:00,408 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/3eee82f00e8948e992f9b72229b35751 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/B/3eee82f00e8948e992f9b72229b35751 2024-11-08T02:31:00,410 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1d1eebaf90f24e80a0d169df881e8a26 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/1d1eebaf90f24e80a0d169df881e8a26 2024-11-08T02:31:00,412 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/33c610e76c0340438f68e69f2bfcc7eb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/33c610e76c0340438f68e69f2bfcc7eb 2024-11-08T02:31:00,412 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/59f49fd6b62b4aa1967dc843f1384049 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/C/59f49fd6b62b4aa1967dc843f1384049 2024-11-08T02:31:00,415 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/recovered.edits/481.seqid to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9/recovered.edits/481.seqid 2024-11-08T02:31:00,416 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/dc8ad63d9a6a95258392be1a735a16e9 2024-11-08T02:31:00,416 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:31:00,420 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=34, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:00,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-08T02:31:00,426 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:31:00,452 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:31:00,454 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=34, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:00,454 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:31:00,454 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731033060454"}]},"ts":"9223372036854775807"} 2024-11-08T02:31:00,457 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-08T02:31:00,457 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => dc8ad63d9a6a95258392be1a735a16e9, NAME => 'TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:31:00,457 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:31:00,458 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731033060457"}]},"ts":"9223372036854775807"} 2024-11-08T02:31:00,460 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-08T02:31:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-08T02:31:00,503 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=34, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:00,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 120 msec 2024-11-08T02:31:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-08T02:31:00,693 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 34 completed 2024-11-08T02:31:00,708 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMixedAtomicity Thread=239 (was 219) Potentially hanging thread: hconnection-0x616dbb2c-shared-pool-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x616dbb2c-shared-pool-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ecdbff65922a:40577-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_885003772_22 at /127.0.0.1:44938 [Waiting for operation #255] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_45780746_22 at /127.0.0.1:33546 [Waiting for operation #273] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x616dbb2c-shared-pool-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_885003772_22 at /127.0.0.1:48438 [Waiting for operation #123] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x616dbb2c-shared-pool-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=458 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=238 (was 117) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2319 (was 2864) 2024-11-08T02:31:00,717 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMobMixedAtomicity Thread=239, OpenFileDescriptor=458, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=2318 2024-11-08T02:31:00,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:31:00,720 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:31:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=35, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:00,722 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=35, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:31:00,722 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:00,722 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 35 2024-11-08T02:31:00,723 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=35, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:31:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=35 2024-11-08T02:31:00,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741944_1120 (size=960) 2024-11-08T02:31:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=35 2024-11-08T02:31:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=35 2024-11-08T02:31:01,137 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:31:01,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741945_1121 (size=53) 2024-11-08T02:31:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=35 2024-11-08T02:31:01,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:31:01,548 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 8b6bdcd9b78ee6cd044820b0655675ec, disabling compactions & flushes 2024-11-08T02:31:01,548 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. after waiting 0 ms 2024-11-08T02:31:01,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:01,551 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=35, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:31:01,552 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731033061552"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731033061552"}]},"ts":"1731033061552"} 2024-11-08T02:31:01,555 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T02:31:01,557 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=35, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:31:01,557 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033061557"}]},"ts":"1731033061557"} 2024-11-08T02:31:01,559 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:31:01,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, ASSIGN}] 2024-11-08T02:31:01,613 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=36, ppid=35, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, ASSIGN 2024-11-08T02:31:01,615 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=36, ppid=35, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, ASSIGN; state=OFFLINE, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=false 2024-11-08T02:31:01,766 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=36 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:01,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; OpenRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:31:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=35 2024-11-08T02:31:01,923 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:01,930 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,931 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7285): Opening region: {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:31:01,932 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,932 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:31:01,932 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7327): checking encryption for 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,933 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7330): checking classloading for 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,935 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,936 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:01,936 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6bdcd9b78ee6cd044820b0655675ec columnFamilyName A 2024-11-08T02:31:01,936 DEBUG [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:01,937 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(327): Store=8b6bdcd9b78ee6cd044820b0655675ec/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:01,937 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,938 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:01,939 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6bdcd9b78ee6cd044820b0655675ec columnFamilyName B 2024-11-08T02:31:01,939 DEBUG [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:01,939 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(327): Store=8b6bdcd9b78ee6cd044820b0655675ec/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:01,939 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,940 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:01,941 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6bdcd9b78ee6cd044820b0655675ec columnFamilyName C 2024-11-08T02:31:01,941 DEBUG [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:01,941 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(327): Store=8b6bdcd9b78ee6cd044820b0655675ec/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:01,941 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,942 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,942 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,944 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:31:01,945 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1085): writing seq id for 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:01,948 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:31:01,949 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1102): Opened 8b6bdcd9b78ee6cd044820b0655675ec; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66622447, jitterRate=-0.007248178124427795}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:31:01,951 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1001): Region open journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:01,952 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., pid=37, masterSystemTime=1731033061923 2024-11-08T02:31:01,953 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,953 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:01,954 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=36 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:01,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T02:31:01,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, ppid=36, state=SUCCESS; OpenRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 in 186 msec 2024-11-08T02:31:01,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=36, resume processing ppid=35 2024-11-08T02:31:01,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, ppid=35, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, ASSIGN in 345 msec 2024-11-08T02:31:01,959 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=35, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:31:01,959 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033061959"}]},"ts":"1731033061959"} 2024-11-08T02:31:01,960 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:31:01,976 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=35, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:31:01,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2560 sec 2024-11-08T02:31:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=35 2024-11-08T02:31:02,836 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 35 completed 2024-11-08T02:31:02,842 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x118b007e to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d29de25 2024-11-08T02:31:02,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a378df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:02,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:02,894 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46474, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:02,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:31:02,899 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60962, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:31:02,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:31:02,907 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:31:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=38, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:02,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741946_1122 (size=996) 2024-11-08T02:31:03,328 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.960 2024-11-08T02:31:03,328 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.996 2024-11-08T02:31:03,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:31:03,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, REOPEN/MOVE}] 2024-11-08T02:31:03,343 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, REOPEN/MOVE 2024-11-08T02:31:03,344 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:03,345 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:31:03,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; CloseRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:31:03,497 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:03,498 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(124): Close 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,498 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:31:03,498 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1681): Closing 8b6bdcd9b78ee6cd044820b0655675ec, disabling compactions & flushes 2024-11-08T02:31:03,498 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,499 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,499 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. after waiting 0 ms 2024-11-08T02:31:03,499 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,508 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T02:31:03,510 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,510 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegion(1635): Region close journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:03,510 WARN [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] regionserver.HRegionServer(3786): Not adding moved region record: 8b6bdcd9b78ee6cd044820b0655675ec to self. 2024-11-08T02:31:03,513 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=CLOSED 2024-11-08T02:31:03,513 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=41}] handler.UnassignRegionHandler(170): Closed 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T02:31:03,516 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, REOPEN/MOVE; state=CLOSED, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=true 2024-11-08T02:31:03,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; CloseRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 in 169 msec 2024-11-08T02:31:03,667 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:03,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=40, state=RUNNABLE; OpenRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:31:03,823 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:03,827 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,827 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7285): Opening region: {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:31:03,828 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,828 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:31:03,828 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7327): checking encryption for 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,828 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7330): checking classloading for 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,835 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,836 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:03,841 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6bdcd9b78ee6cd044820b0655675ec columnFamilyName A 2024-11-08T02:31:03,843 DEBUG [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:03,843 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(327): Store=8b6bdcd9b78ee6cd044820b0655675ec/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:03,844 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,844 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:03,845 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6bdcd9b78ee6cd044820b0655675ec columnFamilyName B 2024-11-08T02:31:03,845 DEBUG [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:03,845 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(327): Store=8b6bdcd9b78ee6cd044820b0655675ec/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:03,845 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,846 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:03,846 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b6bdcd9b78ee6cd044820b0655675ec columnFamilyName C 2024-11-08T02:31:03,846 DEBUG [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:03,846 INFO [StoreOpener-8b6bdcd9b78ee6cd044820b0655675ec-1 {}] regionserver.HStore(327): Store=8b6bdcd9b78ee6cd044820b0655675ec/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:03,846 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,847 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,848 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,849 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:31:03,850 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1085): writing seq id for 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:03,851 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1102): Opened 8b6bdcd9b78ee6cd044820b0655675ec; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60468172, jitterRate=-0.09895402193069458}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:31:03,852 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1001): Region open journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:03,852 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., pid=42, masterSystemTime=1731033063823 2024-11-08T02:31:03,854 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,854 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:03,854 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=OPEN, openSeqNum=5, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:03,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=40 2024-11-08T02:31:03,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=40, state=SUCCESS; OpenRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 in 186 msec 2024-11-08T02:31:03,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=40, resume processing ppid=39 2024-11-08T02:31:03,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, REOPEN/MOVE in 515 msec 2024-11-08T02:31:03,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=39, resume processing ppid=38 2024-11-08T02:31:03,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, ppid=38, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 527 msec 2024-11-08T02:31:03,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 952 msec 2024-11-08T02:31:03,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=38 2024-11-08T02:31:03,871 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x79982672 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2931c73e 2024-11-08T02:31:03,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bad2e85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:03,936 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b4bd1ba to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@491ea2ee 2024-11-08T02:31:03,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328f994d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:03,955 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x454f1431 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@190853fc 2024-11-08T02:31:03,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9306be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:03,970 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x505d5ccd to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46114993 2024-11-08T02:31:03,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465dc764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:03,988 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x367f47f7 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2885d2d9 2024-11-08T02:31:03,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cb464a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:03,996 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22e911df to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@78cafade 2024-11-08T02:31:04,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@152377d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:04,004 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3b727d6e to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14c16cd4 2024-11-08T02:31:04,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a52344f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:04,012 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1c7940d9 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@341384e 2024-11-08T02:31:04,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8ba8425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:04,022 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c38ee58 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26b120d9 2024-11-08T02:31:04,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af61386, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:04,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=43, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=43, table=TestAcidGuarantees 2024-11-08T02:31:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-08T02:31:04,038 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=43, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=43, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:04,039 DEBUG [hconnection-0x5e3f788c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=43, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=43, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:04,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:04,043 DEBUG [hconnection-0x7dcb8389-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,044 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46486, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,045 DEBUG [hconnection-0x28d3a860-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,045 DEBUG [hconnection-0x237bfb8a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,046 DEBUG [hconnection-0x142e1b61-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,046 DEBUG [hconnection-0x43a17196-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,046 DEBUG [hconnection-0x3a16a465-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,047 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46506, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,047 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46532, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,047 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46504, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,047 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46502, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,047 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46538, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,048 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46552, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,048 DEBUG [hconnection-0x5ad6f80e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,049 DEBUG [hconnection-0x67dc70f8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:04,050 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46566, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,050 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46556, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:04,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:31:04,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:04,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:04,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:04,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033124100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033124096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033124110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033124111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033124113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082307278ea2ab4340ad87b4d1bb1e023b_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033064056/Put/seqid=0 2024-11-08T02:31:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-08T02:31:04,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741947_1123 (size=12154) 2024-11-08T02:31:04,145 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:04,151 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082307278ea2ab4340ad87b4d1bb1e023b_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082307278ea2ab4340ad87b4d1bb1e023b_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:04,153 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e228c787be3e4d43ae63ef33d3a559e8, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:04,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e228c787be3e4d43ae63ef33d3a559e8 is 175, key is test_row_0/A:col10/1731033064056/Put/seqid=0 2024-11-08T02:31:04,190 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=44 2024-11-08T02:31:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] handler.RSProcedureHandler(58): pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741948_1124 (size=30955) 2024-11-08T02:31:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=44 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033124214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033124215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033124216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033124216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033124217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,333 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T02:31:04,334 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60978, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T02:31:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-08T02:31:04,344 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=44 2024-11-08T02:31:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] handler.RSProcedureHandler(58): pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=44 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033124424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033124424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033124424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033124424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033124425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,497 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=44 2024-11-08T02:31:04,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:04,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] handler.RSProcedureHandler(58): pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=44 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e228c787be3e4d43ae63ef33d3a559e8 2024-11-08T02:31:04,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e4d22ac4bde941eb8eab802595750aa1 is 50, key is test_row_0/B:col10/1731033064056/Put/seqid=0 2024-11-08T02:31:04,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741949_1125 (size=12001) 2024-11-08T02:31:04,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e4d22ac4bde941eb8eab802595750aa1 2024-11-08T02:31:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-08T02:31:04,650 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=44 2024-11-08T02:31:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] handler.RSProcedureHandler(58): pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=44 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/e608f68f70544d4fa12b8ff498526a34 is 50, key is test_row_0/C:col10/1731033064056/Put/seqid=0 2024-11-08T02:31:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741950_1126 (size=12001) 2024-11-08T02:31:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033124730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033124731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033124731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033124732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:04,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033124733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,805 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=44 2024-11-08T02:31:04,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:04,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] handler.RSProcedureHandler(58): pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=44 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,958 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:04,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=44 2024-11-08T02:31:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:04,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] handler.RSProcedureHandler(58): pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=44 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=44 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:05,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/e608f68f70544d4fa12b8ff498526a34 2024-11-08T02:31:05,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e228c787be3e4d43ae63ef33d3a559e8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e228c787be3e4d43ae63ef33d3a559e8 2024-11-08T02:31:05,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e228c787be3e4d43ae63ef33d3a559e8, entries=150, sequenceid=16, filesize=30.2 K 2024-11-08T02:31:05,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e4d22ac4bde941eb8eab802595750aa1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d22ac4bde941eb8eab802595750aa1 2024-11-08T02:31:05,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d22ac4bde941eb8eab802595750aa1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-08T02:31:05,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/e608f68f70544d4fa12b8ff498526a34 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/e608f68f70544d4fa12b8ff498526a34 2024-11-08T02:31:05,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/e608f68f70544d4fa12b8ff498526a34, entries=150, sequenceid=16, filesize=11.7 K 2024-11-08T02:31:05,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8b6bdcd9b78ee6cd044820b0655675ec in 1041ms, sequenceid=16, compaction requested=false 2024-11-08T02:31:05,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:05,111 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=44 2024-11-08T02:31:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:05,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:31:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:05,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:05,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d09d73ecae54474aaa9bb4383085c27d_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033064092/Put/seqid=0 2024-11-08T02:31:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-08T02:31:05,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741951_1127 (size=12154) 2024-11-08T02:31:05,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:05,178 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d09d73ecae54474aaa9bb4383085c27d_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d09d73ecae54474aaa9bb4383085c27d_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:05,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6c3f90ba12f4feb8bf299d56a3a33ba, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:05,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6c3f90ba12f4feb8bf299d56a3a33ba is 175, key is test_row_0/A:col10/1731033064092/Put/seqid=0 2024-11-08T02:31:05,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741952_1128 (size=30955) 2024-11-08T02:31:05,188 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6c3f90ba12f4feb8bf299d56a3a33ba 2024-11-08T02:31:05,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/8f291cab6af04cbe90aef95b41c8d398 is 50, key is test_row_0/B:col10/1731033064092/Put/seqid=0 2024-11-08T02:31:05,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741953_1129 (size=12001) 2024-11-08T02:31:05,220 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/8f291cab6af04cbe90aef95b41c8d398 2024-11-08T02:31:05,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/fc803009282e42478ab19dc7af7d75d3 is 50, key is test_row_0/C:col10/1731033064092/Put/seqid=0 2024-11-08T02:31:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:05,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033125245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741954_1130 (size=12001) 2024-11-08T02:31:05,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033125249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033125251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033125250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033125251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033125352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033125355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033125356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033125356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033125356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,436 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:31:05,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033125555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033125561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033125561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033125562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033125562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,654 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/fc803009282e42478ab19dc7af7d75d3 2024-11-08T02:31:05,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6c3f90ba12f4feb8bf299d56a3a33ba as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6c3f90ba12f4feb8bf299d56a3a33ba 2024-11-08T02:31:05,667 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6c3f90ba12f4feb8bf299d56a3a33ba, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T02:31:05,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/8f291cab6af04cbe90aef95b41c8d398 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/8f291cab6af04cbe90aef95b41c8d398 2024-11-08T02:31:05,674 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/8f291cab6af04cbe90aef95b41c8d398, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:31:05,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/fc803009282e42478ab19dc7af7d75d3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fc803009282e42478ab19dc7af7d75d3 2024-11-08T02:31:05,687 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fc803009282e42478ab19dc7af7d75d3, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:31:05,688 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 8b6bdcd9b78ee6cd044820b0655675ec in 576ms, sequenceid=40, compaction requested=false 2024-11-08T02:31:05,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:05,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:05,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44 2024-11-08T02:31:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=44 2024-11-08T02:31:05,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=43 2024-11-08T02:31:05,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6510 sec 2024-11-08T02:31:05,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=43, table=TestAcidGuarantees in 1.6560 sec 2024-11-08T02:31:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:05,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:31:05,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:05,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:05,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:05,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:05,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:05,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:05,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110873d8de4f6dd14c3281cb1627d463c97a_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033065857/Put/seqid=0 2024-11-08T02:31:05,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741955_1131 (size=14594) 2024-11-08T02:31:05,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:05,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110873d8de4f6dd14c3281cb1627d463c97a_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110873d8de4f6dd14c3281cb1627d463c97a_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:05,891 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/b420f3672b4b40fc8e1e2a35be5ddd33, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:05,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/b420f3672b4b40fc8e1e2a35be5ddd33 is 175, key is test_row_0/A:col10/1731033065857/Put/seqid=0 2024-11-08T02:31:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033125889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033125890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033125899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033125900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033125900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:05,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741956_1132 (size=39549) 2024-11-08T02:31:05,914 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/b420f3672b4b40fc8e1e2a35be5ddd33 2024-11-08T02:31:05,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/974c8b300d004c50a4c32bb79a67a734 is 50, key is test_row_0/B:col10/1731033065857/Put/seqid=0 2024-11-08T02:31:05,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741957_1133 (size=12001) 2024-11-08T02:31:05,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/974c8b300d004c50a4c32bb79a67a734 2024-11-08T02:31:05,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/1fdfb1c0498e4ebead1a495f67665a89 is 50, key is test_row_0/C:col10/1731033065857/Put/seqid=0 2024-11-08T02:31:05,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741958_1134 (size=12001) 2024-11-08T02:31:06,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033126001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033126001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033126005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033126006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033126006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-08T02:31:06,143 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 43 completed 2024-11-08T02:31:06,144 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees 2024-11-08T02:31:06,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T02:31:06,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=45, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:06,147 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=45, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:06,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:06,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033126205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033126207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033126209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033126211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033126211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T02:31:06,299 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-08T02:31:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:06,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] handler.RSProcedureHandler(58): pid=46 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=46 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=46 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:06,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/1fdfb1c0498e4ebead1a495f67665a89 2024-11-08T02:31:06,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/b420f3672b4b40fc8e1e2a35be5ddd33 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/b420f3672b4b40fc8e1e2a35be5ddd33 2024-11-08T02:31:06,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/b420f3672b4b40fc8e1e2a35be5ddd33, entries=200, sequenceid=54, filesize=38.6 K 2024-11-08T02:31:06,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/974c8b300d004c50a4c32bb79a67a734 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/974c8b300d004c50a4c32bb79a67a734 2024-11-08T02:31:06,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/974c8b300d004c50a4c32bb79a67a734, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:31:06,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/1fdfb1c0498e4ebead1a495f67665a89 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/1fdfb1c0498e4ebead1a495f67665a89 2024-11-08T02:31:06,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/1fdfb1c0498e4ebead1a495f67665a89, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:31:06,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8b6bdcd9b78ee6cd044820b0655675ec in 540ms, sequenceid=54, compaction requested=true 2024-11-08T02:31:06,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:06,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:06,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:06,399 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:06,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:06,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:06,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:06,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:06,399 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:06,401 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:06,401 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/A is initiating minor compaction (all files) 2024-11-08T02:31:06,401 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/A in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:06,401 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e228c787be3e4d43ae63ef33d3a559e8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6c3f90ba12f4feb8bf299d56a3a33ba, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/b420f3672b4b40fc8e1e2a35be5ddd33] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=99.1 K 2024-11-08T02:31:06,401 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:06,401 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e228c787be3e4d43ae63ef33d3a559e8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6c3f90ba12f4feb8bf299d56a3a33ba, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/b420f3672b4b40fc8e1e2a35be5ddd33] 2024-11-08T02:31:06,401 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:06,402 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/B is initiating minor compaction (all files) 2024-11-08T02:31:06,402 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/B in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:06,402 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting e228c787be3e4d43ae63ef33d3a559e8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731033064056 2024-11-08T02:31:06,402 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d22ac4bde941eb8eab802595750aa1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/8f291cab6af04cbe90aef95b41c8d398, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/974c8b300d004c50a4c32bb79a67a734] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.2 K 2024-11-08T02:31:06,402 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting a6c3f90ba12f4feb8bf299d56a3a33ba, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033064092 2024-11-08T02:31:06,402 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e4d22ac4bde941eb8eab802595750aa1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731033064056 2024-11-08T02:31:06,403 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b420f3672b4b40fc8e1e2a35be5ddd33, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033065247 2024-11-08T02:31:06,403 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 8f291cab6af04cbe90aef95b41c8d398, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033064092 2024-11-08T02:31:06,408 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 974c8b300d004c50a4c32bb79a67a734, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033065249 2024-11-08T02:31:06,429 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:06,431 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:06,432 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/44515627f9934dbab055261d9dbaacdf is 50, key is test_row_0/B:col10/1731033065857/Put/seqid=0 2024-11-08T02:31:06,434 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110888ec74f249ff406c98b471b1ad672599_8b6bdcd9b78ee6cd044820b0655675ec store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:06,441 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110888ec74f249ff406c98b471b1ad672599_8b6bdcd9b78ee6cd044820b0655675ec, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:06,441 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110888ec74f249ff406c98b471b1ad672599_8b6bdcd9b78ee6cd044820b0655675ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T02:31:06,452 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741959_1135 (size=12104) 2024-11-08T02:31:06,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-08T02:31:06,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:06,453 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:31:06,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:06,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:06,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:06,460 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/44515627f9934dbab055261d9dbaacdf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/44515627f9934dbab055261d9dbaacdf 2024-11-08T02:31:06,467 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/B of 8b6bdcd9b78ee6cd044820b0655675ec into 44515627f9934dbab055261d9dbaacdf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:06,467 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:06,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741960_1136 (size=4469) 2024-11-08T02:31:06,467 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/B, priority=13, startTime=1731033066399; duration=0sec 2024-11-08T02:31:06,467 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:06,468 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:B 2024-11-08T02:31:06,468 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:06,468 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:06,468 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/C is initiating minor compaction (all files) 2024-11-08T02:31:06,469 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/C in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:06,469 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/e608f68f70544d4fa12b8ff498526a34, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fc803009282e42478ab19dc7af7d75d3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/1fdfb1c0498e4ebead1a495f67665a89] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.2 K 2024-11-08T02:31:06,469 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e608f68f70544d4fa12b8ff498526a34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731033064056 2024-11-08T02:31:06,471 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting fc803009282e42478ab19dc7af7d75d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033064092 2024-11-08T02:31:06,471 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#A#compaction#115 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:06,471 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1fdfb1c0498e4ebead1a495f67665a89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033065249 2024-11-08T02:31:06,473 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/83ef3682c1fd4748905f0cdb8ad3f71d is 175, key is test_row_0/A:col10/1731033065857/Put/seqid=0 2024-11-08T02:31:06,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bda8eecf7e3d45a493f703f6457b4437_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033065898/Put/seqid=0 2024-11-08T02:31:06,500 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#C#compaction#117 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:06,501 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/aa72908f197a47d7b28cd5ceaf65eec8 is 50, key is test_row_0/C:col10/1731033065857/Put/seqid=0 2024-11-08T02:31:06,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741961_1137 (size=31058) 2024-11-08T02:31:06,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:06,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:06,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741962_1138 (size=12154) 2024-11-08T02:31:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033126522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033126526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,535 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bda8eecf7e3d45a493f703f6457b4437_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bda8eecf7e3d45a493f703f6457b4437_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033126527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033126528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033126529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/cf8e74a2c0d54cb1a019c2f7a59503a1, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:06,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/cf8e74a2c0d54cb1a019c2f7a59503a1 is 175, key is test_row_0/A:col10/1731033065898/Put/seqid=0 2024-11-08T02:31:06,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741963_1139 (size=12104) 2024-11-08T02:31:06,552 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/aa72908f197a47d7b28cd5ceaf65eec8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/aa72908f197a47d7b28cd5ceaf65eec8 2024-11-08T02:31:06,565 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/C of 8b6bdcd9b78ee6cd044820b0655675ec into aa72908f197a47d7b28cd5ceaf65eec8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:06,565 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:06,565 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/C, priority=13, startTime=1731033066399; duration=0sec 2024-11-08T02:31:06,565 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:06,565 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:C 2024-11-08T02:31:06,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741964_1140 (size=30955) 2024-11-08T02:31:06,573 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/cf8e74a2c0d54cb1a019c2f7a59503a1 2024-11-08T02:31:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/5d0354565eb749f0be4dbd6f723b092c is 50, key is test_row_0/B:col10/1731033065898/Put/seqid=0 2024-11-08T02:31:06,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741965_1141 (size=12001) 2024-11-08T02:31:06,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033126629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033126636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033126636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033126636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033126637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T02:31:06,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033126832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033126839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033126840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033126841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:06,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033126842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:06,913 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/83ef3682c1fd4748905f0cdb8ad3f71d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/83ef3682c1fd4748905f0cdb8ad3f71d 2024-11-08T02:31:06,919 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/A of 8b6bdcd9b78ee6cd044820b0655675ec into 83ef3682c1fd4748905f0cdb8ad3f71d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:06,919 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:06,919 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/A, priority=13, startTime=1731033066399; duration=0sec 2024-11-08T02:31:06,919 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:06,919 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:A 2024-11-08T02:31:06,993 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/5d0354565eb749f0be4dbd6f723b092c 2024-11-08T02:31:07,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/932dee5655b440baaee7312d6875d323 is 50, key is test_row_0/C:col10/1731033065898/Put/seqid=0 2024-11-08T02:31:07,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741966_1142 (size=12001) 2024-11-08T02:31:07,010 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/932dee5655b440baaee7312d6875d323 2024-11-08T02:31:07,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/cf8e74a2c0d54cb1a019c2f7a59503a1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/cf8e74a2c0d54cb1a019c2f7a59503a1 2024-11-08T02:31:07,023 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/cf8e74a2c0d54cb1a019c2f7a59503a1, entries=150, sequenceid=76, filesize=30.2 K 2024-11-08T02:31:07,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/5d0354565eb749f0be4dbd6f723b092c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5d0354565eb749f0be4dbd6f723b092c 2024-11-08T02:31:07,029 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5d0354565eb749f0be4dbd6f723b092c, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:31:07,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/932dee5655b440baaee7312d6875d323 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/932dee5655b440baaee7312d6875d323 2024-11-08T02:31:07,035 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/932dee5655b440baaee7312d6875d323, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:31:07,036 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 8b6bdcd9b78ee6cd044820b0655675ec in 583ms, sequenceid=76, compaction requested=false 2024-11-08T02:31:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-11-08T02:31:07,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=46 2024-11-08T02:31:07,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-08T02:31:07,039 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 891 msec 2024-11-08T02:31:07,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees in 896 msec 2024-11-08T02:31:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:07,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:31:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:07,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:07,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087b090aaf07b24bd48bdc6170e36f8f19_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033066518/Put/seqid=0 2024-11-08T02:31:07,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741967_1143 (size=12154) 2024-11-08T02:31:07,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033127160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033127160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033127161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033127161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033127162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-08T02:31:07,252 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 45 completed 2024-11-08T02:31:07,254 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:07,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees 2024-11-08T02:31:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-08T02:31:07,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=47, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:07,255 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=47, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:07,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033127264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033127265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033127265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033127265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033127266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-08T02:31:07,407 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-08T02:31:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033127468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033127468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033127469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033127470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033127470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-08T02:31:07,557 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:07,560 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-08T02:31:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,561 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087b090aaf07b24bd48bdc6170e36f8f19_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087b090aaf07b24bd48bdc6170e36f8f19_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,562 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e9b70d4a6fc747f1a8426fc0547f382e, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:07,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e9b70d4a6fc747f1a8426fc0547f382e is 175, key is test_row_0/A:col10/1731033066518/Put/seqid=0 2024-11-08T02:31:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741968_1144 (size=30955) 2024-11-08T02:31:07,713 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-08T02:31:07,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033127772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033127772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033127772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033127772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:07,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033127773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-08T02:31:07,866 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:07,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-08T02:31:07,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:07,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:07,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:07,968 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e9b70d4a6fc747f1a8426fc0547f382e 2024-11-08T02:31:07,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/c020daa3856e468fbc301d79feeaf760 is 50, key is test_row_0/B:col10/1731033066518/Put/seqid=0 2024-11-08T02:31:07,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741969_1145 (size=12001) 2024-11-08T02:31:07,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/c020daa3856e468fbc301d79feeaf760 2024-11-08T02:31:07,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b4ec00cc7feb469a9746ca609274daed is 50, key is test_row_0/C:col10/1731033066518/Put/seqid=0 2024-11-08T02:31:07,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741970_1146 (size=12001) 2024-11-08T02:31:07,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b4ec00cc7feb469a9746ca609274daed 2024-11-08T02:31:07,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/e9b70d4a6fc747f1a8426fc0547f382e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e9b70d4a6fc747f1a8426fc0547f382e 2024-11-08T02:31:08,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e9b70d4a6fc747f1a8426fc0547f382e, entries=150, sequenceid=95, filesize=30.2 K 2024-11-08T02:31:08,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/c020daa3856e468fbc301d79feeaf760 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/c020daa3856e468fbc301d79feeaf760 2024-11-08T02:31:08,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/c020daa3856e468fbc301d79feeaf760, entries=150, sequenceid=95, filesize=11.7 K 2024-11-08T02:31:08,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b4ec00cc7feb469a9746ca609274daed as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4ec00cc7feb469a9746ca609274daed 2024-11-08T02:31:08,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4ec00cc7feb469a9746ca609274daed, entries=150, sequenceid=95, filesize=11.7 K 2024-11-08T02:31:08,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8b6bdcd9b78ee6cd044820b0655675ec in 874ms, sequenceid=95, compaction requested=true 2024-11-08T02:31:08,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:08,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:08,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:08,014 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:08,014 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:08,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:08,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:08,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:08,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:08,015 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:08,015 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:08,016 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/A is initiating minor compaction (all files) 2024-11-08T02:31:08,016 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/B is initiating minor compaction (all files) 2024-11-08T02:31:08,016 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/A in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:08,016 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/B in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:08,016 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/44515627f9934dbab055261d9dbaacdf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5d0354565eb749f0be4dbd6f723b092c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/c020daa3856e468fbc301d79feeaf760] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.3 K 2024-11-08T02:31:08,016 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/83ef3682c1fd4748905f0cdb8ad3f71d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/cf8e74a2c0d54cb1a019c2f7a59503a1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e9b70d4a6fc747f1a8426fc0547f382e] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=90.8 K 2024-11-08T02:31:08,016 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:08,016 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/83ef3682c1fd4748905f0cdb8ad3f71d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/cf8e74a2c0d54cb1a019c2f7a59503a1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e9b70d4a6fc747f1a8426fc0547f382e] 2024-11-08T02:31:08,016 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 44515627f9934dbab055261d9dbaacdf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033065249 2024-11-08T02:31:08,016 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 83ef3682c1fd4748905f0cdb8ad3f71d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033065249 2024-11-08T02:31:08,017 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 5d0354565eb749f0be4dbd6f723b092c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033065888 2024-11-08T02:31:08,017 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting cf8e74a2c0d54cb1a019c2f7a59503a1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033065888 2024-11-08T02:31:08,017 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c020daa3856e468fbc301d79feeaf760, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731033066518 2024-11-08T02:31:08,018 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting e9b70d4a6fc747f1a8426fc0547f382e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731033066518 2024-11-08T02:31:08,018 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-08T02:31:08,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:08,019 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:31:08,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:08,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:08,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:08,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:08,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089a4e25fbfe5949fba90fd2d0c7edc0ae_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033067161/Put/seqid=0 2024-11-08T02:31:08,035 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:08,036 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#B#compaction#124 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:08,037 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/a90d31aea9564b2d9562e180fb113689 is 50, key is test_row_0/B:col10/1731033066518/Put/seqid=0 2024-11-08T02:31:08,045 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411080475619b7c8c400da4e22441254774cf_8b6bdcd9b78ee6cd044820b0655675ec store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:08,050 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411080475619b7c8c400da4e22441254774cf_8b6bdcd9b78ee6cd044820b0655675ec, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:08,050 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080475619b7c8c400da4e22441254774cf_8b6bdcd9b78ee6cd044820b0655675ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:08,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741971_1147 (size=12154) 2024-11-08T02:31:08,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741972_1148 (size=12207) 2024-11-08T02:31:08,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741973_1149 (size=4469) 2024-11-08T02:31:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:08,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033128292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033128292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033128294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033128294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033128294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-08T02:31:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033128400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033128400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033128401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033128401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033128401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:08,460 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089a4e25fbfe5949fba90fd2d0c7edc0ae_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a4e25fbfe5949fba90fd2d0c7edc0ae_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:08,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6ecc0de2fdc4f2b852b9cb5766e1b78, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:08,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6ecc0de2fdc4f2b852b9cb5766e1b78 is 175, key is test_row_0/A:col10/1731033067161/Put/seqid=0 2024-11-08T02:31:08,468 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/a90d31aea9564b2d9562e180fb113689 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/a90d31aea9564b2d9562e180fb113689 2024-11-08T02:31:08,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741974_1150 (size=30955) 2024-11-08T02:31:08,470 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6ecc0de2fdc4f2b852b9cb5766e1b78 2024-11-08T02:31:08,470 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#A#compaction#125 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:08,471 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/f24a068dd2974cb7b10cdeae41ab3545 is 175, key is test_row_0/A:col10/1731033066518/Put/seqid=0 2024-11-08T02:31:08,477 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/B of 8b6bdcd9b78ee6cd044820b0655675ec into a90d31aea9564b2d9562e180fb113689(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:08,477 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:08,477 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/B, priority=13, startTime=1731033068014; duration=0sec 2024-11-08T02:31:08,477 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:08,477 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:B 2024-11-08T02:31:08,477 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:08,479 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:08,479 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/C is initiating minor compaction (all files) 2024-11-08T02:31:08,479 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/C in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:08,479 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/aa72908f197a47d7b28cd5ceaf65eec8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/932dee5655b440baaee7312d6875d323, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4ec00cc7feb469a9746ca609274daed] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.3 K 2024-11-08T02:31:08,480 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting aa72908f197a47d7b28cd5ceaf65eec8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033065249 2024-11-08T02:31:08,480 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 932dee5655b440baaee7312d6875d323, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033065888 2024-11-08T02:31:08,481 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b4ec00cc7feb469a9746ca609274daed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731033066518 2024-11-08T02:31:08,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b414012759a642a58e7d69329c90e019 is 50, key is test_row_0/B:col10/1731033067161/Put/seqid=0 2024-11-08T02:31:08,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741975_1151 (size=31161) 2024-11-08T02:31:08,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741976_1152 (size=12001) 2024-11-08T02:31:08,501 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#C#compaction#127 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:08,501 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b4254184686448ed80ef095ca8d83cb6 is 50, key is test_row_0/C:col10/1731033066518/Put/seqid=0 2024-11-08T02:31:08,505 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/f24a068dd2974cb7b10cdeae41ab3545 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f24a068dd2974cb7b10cdeae41ab3545 2024-11-08T02:31:08,518 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/A of 8b6bdcd9b78ee6cd044820b0655675ec into f24a068dd2974cb7b10cdeae41ab3545(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:08,518 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:08,518 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/A, priority=13, startTime=1731033068014; duration=0sec 2024-11-08T02:31:08,518 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:08,518 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:A 2024-11-08T02:31:08,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741977_1153 (size=12207) 2024-11-08T02:31:08,530 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b4254184686448ed80ef095ca8d83cb6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4254184686448ed80ef095ca8d83cb6 2024-11-08T02:31:08,536 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/C of 8b6bdcd9b78ee6cd044820b0655675ec into b4254184686448ed80ef095ca8d83cb6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:08,536 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:08,536 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/C, priority=13, startTime=1731033068015; duration=0sec 2024-11-08T02:31:08,536 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:08,536 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:C 2024-11-08T02:31:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033128602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033128602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033128603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033128604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033128604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,890 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b414012759a642a58e7d69329c90e019 2024-11-08T02:31:08,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/436515431bfa4640a238bd8c18b4e9b3 is 50, key is test_row_0/C:col10/1731033067161/Put/seqid=0 2024-11-08T02:31:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033128906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033128907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033128907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033128909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033128909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:08,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741978_1154 (size=12001) 2024-11-08T02:31:08,924 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/436515431bfa4640a238bd8c18b4e9b3 2024-11-08T02:31:08,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a6ecc0de2fdc4f2b852b9cb5766e1b78 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6ecc0de2fdc4f2b852b9cb5766e1b78 2024-11-08T02:31:08,941 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6ecc0de2fdc4f2b852b9cb5766e1b78, entries=150, sequenceid=115, filesize=30.2 K 2024-11-08T02:31:08,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b414012759a642a58e7d69329c90e019 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b414012759a642a58e7d69329c90e019 2024-11-08T02:31:08,949 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b414012759a642a58e7d69329c90e019, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:31:08,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/436515431bfa4640a238bd8c18b4e9b3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/436515431bfa4640a238bd8c18b4e9b3 2024-11-08T02:31:08,968 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/436515431bfa4640a238bd8c18b4e9b3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:31:08,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 8b6bdcd9b78ee6cd044820b0655675ec in 950ms, sequenceid=115, compaction requested=false 2024-11-08T02:31:08,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:08,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:08,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-08T02:31:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=48 2024-11-08T02:31:08,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47 2024-11-08T02:31:08,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7160 sec 2024-11-08T02:31:08,973 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees in 1.7180 sec 2024-11-08T02:31:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-08T02:31:09,358 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 47 completed 2024-11-08T02:31:09,359 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees 2024-11-08T02:31:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-08T02:31:09,361 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=49, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:09,361 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=49, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:09,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:09,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:09,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:31:09,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:09,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:09,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:09,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:09,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:09,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:09,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085fb425bc58d74422bdb20b85486ef569_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033069412/Put/seqid=0 2024-11-08T02:31:09,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741979_1155 (size=12304) 2024-11-08T02:31:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-08T02:31:09,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033129462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033129462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033129462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033129463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033129463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,512 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033129566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033129566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033129566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033129566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033129566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-08T02:31:09,665 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:09,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:09,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033129768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033129769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033129769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033129770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033129770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,818 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:09,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:09,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:09,842 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085fb425bc58d74422bdb20b85486ef569_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085fb425bc58d74422bdb20b85486ef569_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:09,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/f8697e5b4d43437c82123a7c53cb5609, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:09,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/f8697e5b4d43437c82123a7c53cb5609 is 175, key is test_row_0/A:col10/1731033069412/Put/seqid=0 2024-11-08T02:31:09,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741980_1156 (size=31105) 2024-11-08T02:31:09,854 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/f8697e5b4d43437c82123a7c53cb5609 2024-11-08T02:31:09,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/5199fa2b7a2e4dfda973627fad748ef8 is 50, key is test_row_0/B:col10/1731033069412/Put/seqid=0 2024-11-08T02:31:09,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741981_1157 (size=12151) 2024-11-08T02:31:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-08T02:31:09,970 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:09,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:09,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:09,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:09,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:09,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033130073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033130073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033130073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033130073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033130074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,123 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/5199fa2b7a2e4dfda973627fad748ef8 2024-11-08T02:31:10,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/c1718b581c2745259935085314abbf9a is 50, key is test_row_0/C:col10/1731033069412/Put/seqid=0 2024-11-08T02:31:10,275 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741982_1158 (size=12151) 2024-11-08T02:31:10,428 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-08T02:31:10,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033130576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033130576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033130578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033130578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,581 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:10,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:10,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033130581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/c1718b581c2745259935085314abbf9a 2024-11-08T02:31:10,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/f8697e5b4d43437c82123a7c53cb5609 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f8697e5b4d43437c82123a7c53cb5609 2024-11-08T02:31:10,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f8697e5b4d43437c82123a7c53cb5609, entries=150, sequenceid=137, filesize=30.4 K 2024-11-08T02:31:10,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/5199fa2b7a2e4dfda973627fad748ef8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5199fa2b7a2e4dfda973627fad748ef8 2024-11-08T02:31:10,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5199fa2b7a2e4dfda973627fad748ef8, entries=150, sequenceid=137, filesize=11.9 K 2024-11-08T02:31:10,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/c1718b581c2745259935085314abbf9a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c1718b581c2745259935085314abbf9a 2024-11-08T02:31:10,700 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c1718b581c2745259935085314abbf9a, entries=150, sequenceid=137, filesize=11.9 K 2024-11-08T02:31:10,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 8b6bdcd9b78ee6cd044820b0655675ec in 1288ms, sequenceid=137, compaction requested=true 2024-11-08T02:31:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:10,701 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:10,701 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:10,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:10,702 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:10,702 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:10,702 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/B is initiating minor compaction (all files) 2024-11-08T02:31:10,702 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/A is initiating minor compaction (all files) 2024-11-08T02:31:10,702 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/B in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,702 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/A in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,703 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/a90d31aea9564b2d9562e180fb113689, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b414012759a642a58e7d69329c90e019, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5199fa2b7a2e4dfda973627fad748ef8] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.5 K 2024-11-08T02:31:10,703 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f24a068dd2974cb7b10cdeae41ab3545, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6ecc0de2fdc4f2b852b9cb5766e1b78, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f8697e5b4d43437c82123a7c53cb5609] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=91.0 K 2024-11-08T02:31:10,703 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,703 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f24a068dd2974cb7b10cdeae41ab3545, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6ecc0de2fdc4f2b852b9cb5766e1b78, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f8697e5b4d43437c82123a7c53cb5609] 2024-11-08T02:31:10,703 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a90d31aea9564b2d9562e180fb113689, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731033066518 2024-11-08T02:31:10,703 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f24a068dd2974cb7b10cdeae41ab3545, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731033066518 2024-11-08T02:31:10,703 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b414012759a642a58e7d69329c90e019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033067155 2024-11-08T02:31:10,703 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting a6ecc0de2fdc4f2b852b9cb5766e1b78, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033067155 2024-11-08T02:31:10,704 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f8697e5b4d43437c82123a7c53cb5609, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033068293 2024-11-08T02:31:10,704 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 5199fa2b7a2e4dfda973627fad748ef8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033068293 2024-11-08T02:31:10,709 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:10,711 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:10,711 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/6531c072c7874d21870d7943738adfea is 50, key is test_row_0/B:col10/1731033069412/Put/seqid=0 2024-11-08T02:31:10,712 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411080fab772fd858433b8d7f66dcd4c72a41_8b6bdcd9b78ee6cd044820b0655675ec store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:10,714 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411080fab772fd858433b8d7f66dcd4c72a41_8b6bdcd9b78ee6cd044820b0655675ec, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:10,714 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080fab772fd858433b8d7f66dcd4c72a41_8b6bdcd9b78ee6cd044820b0655675ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:10,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741983_1159 (size=12459) 2024-11-08T02:31:10,723 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/6531c072c7874d21870d7943738adfea as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/6531c072c7874d21870d7943738adfea 2024-11-08T02:31:10,732 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/B of 8b6bdcd9b78ee6cd044820b0655675ec into 6531c072c7874d21870d7943738adfea(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:10,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741984_1160 (size=4469) 2024-11-08T02:31:10,732 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:10,732 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/B, priority=13, startTime=1731033070701; duration=0sec 2024-11-08T02:31:10,732 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:10,732 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:B 2024-11-08T02:31:10,732 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:10,733 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:10,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-08T02:31:10,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,734 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:10,734 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/C is initiating minor compaction (all files) 2024-11-08T02:31:10,734 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:31:10,734 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/C in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:10,734 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4254184686448ed80ef095ca8d83cb6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/436515431bfa4640a238bd8c18b4e9b3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c1718b581c2745259935085314abbf9a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.5 K 2024-11-08T02:31:10,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:10,735 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b4254184686448ed80ef095ca8d83cb6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731033066518 2024-11-08T02:31:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:10,735 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 436515431bfa4640a238bd8c18b4e9b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033067155 2024-11-08T02:31:10,735 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c1718b581c2745259935085314abbf9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033068293 2024-11-08T02:31:10,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ebdd6235d03545e7b187be96b21842c5_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033069461/Put/seqid=0 2024-11-08T02:31:10,753 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#C#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:10,753 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/fea5c0fe49ec4d47b9d2441f5af1e73d is 50, key is test_row_0/C:col10/1731033069412/Put/seqid=0 2024-11-08T02:31:10,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741985_1161 (size=12304) 2024-11-08T02:31:10,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:10,766 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ebdd6235d03545e7b187be96b21842c5_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ebdd6235d03545e7b187be96b21842c5_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:10,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741986_1162 (size=12459) 2024-11-08T02:31:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/d019db358c4b44c2acc0c3e7a0ae0d6d, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:10,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/d019db358c4b44c2acc0c3e7a0ae0d6d is 175, key is test_row_0/A:col10/1731033069461/Put/seqid=0 2024-11-08T02:31:10,773 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/fea5c0fe49ec4d47b9d2441f5af1e73d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fea5c0fe49ec4d47b9d2441f5af1e73d 2024-11-08T02:31:10,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741987_1163 (size=31105) 2024-11-08T02:31:10,778 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/C of 8b6bdcd9b78ee6cd044820b0655675ec into fea5c0fe49ec4d47b9d2441f5af1e73d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:10,778 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:10,779 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/C, priority=13, startTime=1731033070701; duration=0sec 2024-11-08T02:31:10,779 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:10,779 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:C 2024-11-08T02:31:11,134 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#A#compaction#133 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:11,134 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/d0e3c127f3674745b9fd77f73e6a5b88 is 175, key is test_row_0/A:col10/1731033069412/Put/seqid=0 2024-11-08T02:31:11,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741988_1164 (size=31413) 2024-11-08T02:31:11,175 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/d019db358c4b44c2acc0c3e7a0ae0d6d 2024-11-08T02:31:11,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b968cf08fb74431099867d2255704adb is 50, key is test_row_0/B:col10/1731033069461/Put/seqid=0 2024-11-08T02:31:11,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741989_1165 (size=12151) 2024-11-08T02:31:11,186 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b968cf08fb74431099867d2255704adb 2024-11-08T02:31:11,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/f6ceac57e6c44002a628ded3a2fcfd0d is 50, key is test_row_0/C:col10/1731033069461/Put/seqid=0 2024-11-08T02:31:11,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741990_1166 (size=12151) 2024-11-08T02:31:11,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-08T02:31:11,543 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/d0e3c127f3674745b9fd77f73e6a5b88 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d0e3c127f3674745b9fd77f73e6a5b88 2024-11-08T02:31:11,548 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/A of 8b6bdcd9b78ee6cd044820b0655675ec into d0e3c127f3674745b9fd77f73e6a5b88(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:11,548 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:11,548 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/A, priority=13, startTime=1731033070701; duration=0sec 2024-11-08T02:31:11,549 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:11,549 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:A 2024-11-08T02:31:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:11,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:11,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033131594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033131595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033131595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033131595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033131596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,605 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/f6ceac57e6c44002a628ded3a2fcfd0d 2024-11-08T02:31:11,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/d019db358c4b44c2acc0c3e7a0ae0d6d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d019db358c4b44c2acc0c3e7a0ae0d6d 2024-11-08T02:31:11,614 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d019db358c4b44c2acc0c3e7a0ae0d6d, entries=150, sequenceid=156, filesize=30.4 K 2024-11-08T02:31:11,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b968cf08fb74431099867d2255704adb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b968cf08fb74431099867d2255704adb 2024-11-08T02:31:11,619 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b968cf08fb74431099867d2255704adb, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T02:31:11,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/f6ceac57e6c44002a628ded3a2fcfd0d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/f6ceac57e6c44002a628ded3a2fcfd0d 2024-11-08T02:31:11,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/f6ceac57e6c44002a628ded3a2fcfd0d, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T02:31:11,625 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 8b6bdcd9b78ee6cd044820b0655675ec in 891ms, sequenceid=156, compaction requested=false 2024-11-08T02:31:11,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:11,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:11,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-11-08T02:31:11,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-11-08T02:31:11,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-08T02:31:11,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2650 sec 2024-11-08T02:31:11,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees in 2.2690 sec 2024-11-08T02:31:11,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:11,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:31:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:11,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:11,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108eb36585c30194e3790a4f98c5a15e535_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033071702/Put/seqid=0 2024-11-08T02:31:11,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033131711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033131712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033131712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033131713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033131713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741991_1167 (size=17284) 2024-11-08T02:31:11,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033131814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033131815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033131817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033131818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:11,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033131818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033132017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033132018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033132021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033132022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033132022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,123 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:12,128 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108eb36585c30194e3790a4f98c5a15e535_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eb36585c30194e3790a4f98c5a15e535_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:12,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/68996fa3133a4398bc037091828c5ef9, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:12,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/68996fa3133a4398bc037091828c5ef9 is 175, key is test_row_0/A:col10/1731033071702/Put/seqid=0 2024-11-08T02:31:12,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741992_1168 (size=48389) 2024-11-08T02:31:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033132319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033132322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033132323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033132324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033132325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,517 INFO [master/ecdbff65922a:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T02:31:12,517 INFO [master/ecdbff65922a:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T02:31:12,535 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/68996fa3133a4398bc037091828c5ef9 2024-11-08T02:31:12,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b61cb80d384c44cf9c25d318a37b7974 is 50, key is test_row_0/B:col10/1731033071702/Put/seqid=0 2024-11-08T02:31:12,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741993_1169 (size=12151) 2024-11-08T02:31:12,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033132825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033132826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033132828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033132829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033132831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:12,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b61cb80d384c44cf9c25d318a37b7974 2024-11-08T02:31:12,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/a1641cfe169a4f6cb3c46980e9fdaace is 50, key is test_row_0/C:col10/1731033071702/Put/seqid=0 2024-11-08T02:31:12,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741994_1170 (size=12151) 2024-11-08T02:31:13,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/a1641cfe169a4f6cb3c46980e9fdaace 2024-11-08T02:31:13,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/68996fa3133a4398bc037091828c5ef9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/68996fa3133a4398bc037091828c5ef9 2024-11-08T02:31:13,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/68996fa3133a4398bc037091828c5ef9, entries=250, sequenceid=180, filesize=47.3 K 2024-11-08T02:31:13,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b61cb80d384c44cf9c25d318a37b7974 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b61cb80d384c44cf9c25d318a37b7974 2024-11-08T02:31:13,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b61cb80d384c44cf9c25d318a37b7974, entries=150, sequenceid=180, filesize=11.9 K 2024-11-08T02:31:13,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/a1641cfe169a4f6cb3c46980e9fdaace as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/a1641cfe169a4f6cb3c46980e9fdaace 2024-11-08T02:31:13,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/a1641cfe169a4f6cb3c46980e9fdaace, entries=150, sequenceid=180, filesize=11.9 K 2024-11-08T02:31:13,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 8b6bdcd9b78ee6cd044820b0655675ec in 1681ms, sequenceid=180, compaction requested=true 2024-11-08T02:31:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:13,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:13,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:13,383 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:13,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:13,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:13,383 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:13,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:13,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:13,384 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:13,384 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:13,384 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/B is initiating minor compaction (all files) 2024-11-08T02:31:13,384 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/B in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:13,384 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/6531c072c7874d21870d7943738adfea, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b968cf08fb74431099867d2255704adb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b61cb80d384c44cf9c25d318a37b7974] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.9 K 2024-11-08T02:31:13,384 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6531c072c7874d21870d7943738adfea, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033068293 2024-11-08T02:31:13,384 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b968cf08fb74431099867d2255704adb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033069432 2024-11-08T02:31:13,385 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b61cb80d384c44cf9c25d318a37b7974, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731033071595 2024-11-08T02:31:13,386 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/A is initiating minor compaction (all files) 2024-11-08T02:31:13,386 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/A in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:13,387 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d0e3c127f3674745b9fd77f73e6a5b88, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d019db358c4b44c2acc0c3e7a0ae0d6d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/68996fa3133a4398bc037091828c5ef9] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=108.3 K 2024-11-08T02:31:13,387 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:13,387 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d0e3c127f3674745b9fd77f73e6a5b88, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d019db358c4b44c2acc0c3e7a0ae0d6d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/68996fa3133a4398bc037091828c5ef9] 2024-11-08T02:31:13,387 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d0e3c127f3674745b9fd77f73e6a5b88, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033068293 2024-11-08T02:31:13,387 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d019db358c4b44c2acc0c3e7a0ae0d6d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033069432 2024-11-08T02:31:13,388 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 68996fa3133a4398bc037091828c5ef9, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731033071595 2024-11-08T02:31:13,393 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:13,393 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:13,394 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e3be797d5ae94c5c994899ce330dc423 is 50, key is test_row_0/B:col10/1731033071702/Put/seqid=0 2024-11-08T02:31:13,396 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411085f2ae7e28de14cc1bcd9e15a4bfa73b0_8b6bdcd9b78ee6cd044820b0655675ec store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:13,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741995_1171 (size=12561) 2024-11-08T02:31:13,398 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411085f2ae7e28de14cc1bcd9e15a4bfa73b0_8b6bdcd9b78ee6cd044820b0655675ec, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:13,398 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085f2ae7e28de14cc1bcd9e15a4bfa73b0_8b6bdcd9b78ee6cd044820b0655675ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:13,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741996_1172 (size=4469) 2024-11-08T02:31:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-08T02:31:13,465 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 49 completed 2024-11-08T02:31:13,466 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees 2024-11-08T02:31:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T02:31:13,467 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=51, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:13,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=51, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:13,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T02:31:13,619 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=52 2024-11-08T02:31:13,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:13,620 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:31:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108225abf649759410da26729bc75ae1ad2_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033071710/Put/seqid=0 2024-11-08T02:31:13,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741997_1173 (size=12304) 2024-11-08T02:31:13,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T02:31:13,802 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e3be797d5ae94c5c994899ce330dc423 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e3be797d5ae94c5c994899ce330dc423 2024-11-08T02:31:13,804 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#A#compaction#142 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:13,804 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/6d9a3fcb2c3b4991be31bdf4866548ef is 175, key is test_row_0/A:col10/1731033071702/Put/seqid=0 2024-11-08T02:31:13,808 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/B of 8b6bdcd9b78ee6cd044820b0655675ec into e3be797d5ae94c5c994899ce330dc423(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:13,808 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:13,808 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/B, priority=13, startTime=1731033073383; duration=0sec 2024-11-08T02:31:13,809 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:13,809 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:B 2024-11-08T02:31:13,809 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:13,810 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:13,810 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/C is initiating minor compaction (all files) 2024-11-08T02:31:13,810 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/C in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:13,810 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fea5c0fe49ec4d47b9d2441f5af1e73d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/f6ceac57e6c44002a628ded3a2fcfd0d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/a1641cfe169a4f6cb3c46980e9fdaace] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=35.9 K 2024-11-08T02:31:13,810 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting fea5c0fe49ec4d47b9d2441f5af1e73d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033068293 2024-11-08T02:31:13,811 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f6ceac57e6c44002a628ded3a2fcfd0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033069432 2024-11-08T02:31:13,811 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a1641cfe169a4f6cb3c46980e9fdaace, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731033071595 2024-11-08T02:31:13,826 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:13,826 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b6ec26b2d5bc4e3287e75ef84d17ded9 is 50, key is test_row_0/C:col10/1731033071702/Put/seqid=0 2024-11-08T02:31:13,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741998_1174 (size=31515) 2024-11-08T02:31:13,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741999_1175 (size=12561) 2024-11-08T02:31:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:13,838 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b6ec26b2d5bc4e3287e75ef84d17ded9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b6ec26b2d5bc4e3287e75ef84d17ded9 2024-11-08T02:31:13,843 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/C of 8b6bdcd9b78ee6cd044820b0655675ec into b6ec26b2d5bc4e3287e75ef84d17ded9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:13,843 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:13,843 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/C, priority=13, startTime=1731033073383; duration=0sec 2024-11-08T02:31:13,843 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:13,844 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:C 2024-11-08T02:31:13,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033133887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033133887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033133889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033133889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033133892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033133993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033133994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033133994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033133994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:13,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033133998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:14,062 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108225abf649759410da26729bc75ae1ad2_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108225abf649759410da26729bc75ae1ad2_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:14,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/1d649b74253d47dd95a83d179728a7e9, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:14,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/1d649b74253d47dd95a83d179728a7e9 is 175, key is test_row_0/A:col10/1731033071710/Put/seqid=0 2024-11-08T02:31:14,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742000_1176 (size=31105) 2024-11-08T02:31:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T02:31:14,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033134197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033134197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033134197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033134197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033134199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,231 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/6d9a3fcb2c3b4991be31bdf4866548ef as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/6d9a3fcb2c3b4991be31bdf4866548ef 2024-11-08T02:31:14,237 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/A of 8b6bdcd9b78ee6cd044820b0655675ec into 6d9a3fcb2c3b4991be31bdf4866548ef(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:14,237 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:14,237 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/A, priority=13, startTime=1731033073382; duration=0sec 2024-11-08T02:31:14,237 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:14,237 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:A 2024-11-08T02:31:14,468 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/1d649b74253d47dd95a83d179728a7e9 2024-11-08T02:31:14,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/9b6356d8ee4841e8a8b2f9a685ac1da8 is 50, key is test_row_0/B:col10/1731033071710/Put/seqid=0 2024-11-08T02:31:14,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742001_1177 (size=12151) 2024-11-08T02:31:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033134499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033134500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033134500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033134501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:14,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033134504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T02:31:14,878 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/9b6356d8ee4841e8a8b2f9a685ac1da8 2024-11-08T02:31:14,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/9c235f507fcc4b08b7118089401cb41f is 50, key is test_row_0/C:col10/1731033071710/Put/seqid=0 2024-11-08T02:31:14,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742002_1178 (size=12151) 2024-11-08T02:31:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033135002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033135004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:15,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033135004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:15,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033135005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:15,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033135008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:15,290 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/9c235f507fcc4b08b7118089401cb41f 2024-11-08T02:31:15,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/1d649b74253d47dd95a83d179728a7e9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/1d649b74253d47dd95a83d179728a7e9 2024-11-08T02:31:15,299 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/1d649b74253d47dd95a83d179728a7e9, entries=150, sequenceid=195, filesize=30.4 K 2024-11-08T02:31:15,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/9b6356d8ee4841e8a8b2f9a685ac1da8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/9b6356d8ee4841e8a8b2f9a685ac1da8 2024-11-08T02:31:15,304 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/9b6356d8ee4841e8a8b2f9a685ac1da8, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:31:15,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/9c235f507fcc4b08b7118089401cb41f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9c235f507fcc4b08b7118089401cb41f 2024-11-08T02:31:15,309 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9c235f507fcc4b08b7118089401cb41f, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T02:31:15,310 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 8b6bdcd9b78ee6cd044820b0655675ec in 1691ms, sequenceid=195, compaction requested=false 2024-11-08T02:31:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:15,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=52 2024-11-08T02:31:15,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=52 2024-11-08T02:31:15,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-08T02:31:15,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8430 sec 2024-11-08T02:31:15,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees in 1.8470 sec 2024-11-08T02:31:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-08T02:31:15,570 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 51 completed 2024-11-08T02:31:15,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees 2024-11-08T02:31:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-08T02:31:15,573 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=53, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:15,573 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=53, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:15,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-08T02:31:15,724 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:15,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=54 2024-11-08T02:31:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:15,725 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:31:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:15,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:15,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d5b8c7d494264972932db1679209f84b_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033073891/Put/seqid=0 2024-11-08T02:31:15,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742003_1179 (size=12304) 2024-11-08T02:31:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-08T02:31:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:16,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:16,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033136021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033136022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033136023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033136023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033136023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033136124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033136124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033136124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033136125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033136128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:16,140 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d5b8c7d494264972932db1679209f84b_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d5b8c7d494264972932db1679209f84b_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:16,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/28111a23fc7544c19d74f1a1f86077b1, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:16,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/28111a23fc7544c19d74f1a1f86077b1 is 175, key is test_row_0/A:col10/1731033073891/Put/seqid=0 2024-11-08T02:31:16,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742004_1180 (size=31105) 2024-11-08T02:31:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-08T02:31:16,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033136325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033136326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033136328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033136329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033136331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/28111a23fc7544c19d74f1a1f86077b1 2024-11-08T02:31:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e4d4b116283743f6b3ef407ea0840a03 is 50, key is test_row_0/B:col10/1731033073891/Put/seqid=0 2024-11-08T02:31:16,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742005_1181 (size=12151) 2024-11-08T02:31:16,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033136628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033136630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033136630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033136631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033136634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-08T02:31:16,965 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e4d4b116283743f6b3ef407ea0840a03 2024-11-08T02:31:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/9d537a8ad62d4f47b10540d0e9c88893 is 50, key is test_row_0/C:col10/1731033073891/Put/seqid=0 2024-11-08T02:31:16,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742006_1182 (size=12151) 2024-11-08T02:31:17,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033137132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:17,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:17,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033137132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:17,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033137133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033137134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033137136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:17,376 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/9d537a8ad62d4f47b10540d0e9c88893 2024-11-08T02:31:17,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/28111a23fc7544c19d74f1a1f86077b1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/28111a23fc7544c19d74f1a1f86077b1 2024-11-08T02:31:17,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/28111a23fc7544c19d74f1a1f86077b1, entries=150, sequenceid=219, filesize=30.4 K 2024-11-08T02:31:17,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/e4d4b116283743f6b3ef407ea0840a03 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d4b116283743f6b3ef407ea0840a03 2024-11-08T02:31:17,388 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d4b116283743f6b3ef407ea0840a03, entries=150, sequenceid=219, filesize=11.9 K 2024-11-08T02:31:17,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/9d537a8ad62d4f47b10540d0e9c88893 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9d537a8ad62d4f47b10540d0e9c88893 2024-11-08T02:31:17,393 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9d537a8ad62d4f47b10540d0e9c88893, entries=150, sequenceid=219, filesize=11.9 K 2024-11-08T02:31:17,394 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 8b6bdcd9b78ee6cd044820b0655675ec in 1669ms, sequenceid=219, compaction requested=true 2024-11-08T02:31:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:17,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=54 2024-11-08T02:31:17,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=54 2024-11-08T02:31:17,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=54, resume processing ppid=53 2024-11-08T02:31:17,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, ppid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8220 sec 2024-11-08T02:31:17,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees in 1.8250 sec 2024-11-08T02:31:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-08T02:31:17,676 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 53 completed 2024-11-08T02:31:17,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees 2024-11-08T02:31:17,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-08T02:31:17,678 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=55, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:17,678 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=55, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:17,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:17,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-08T02:31:17,830 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:17,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-08T02:31:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:17,830 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:31:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:17,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089af3d5a3326d40bcaf20a483e629d090_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033076022/Put/seqid=0 2024-11-08T02:31:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742007_1183 (size=12304) 2024-11-08T02:31:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-08T02:31:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033138148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033138147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033138148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033138149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033138150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:18,243 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089af3d5a3326d40bcaf20a483e629d090_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089af3d5a3326d40bcaf20a483e629d090_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/2e69b71c338146d9851bac602f05a657, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/2e69b71c338146d9851bac602f05a657 is 175, key is test_row_0/A:col10/1731033076022/Put/seqid=0 2024-11-08T02:31:18,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742008_1184 (size=31105) 2024-11-08T02:31:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033138253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033138253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033138253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033138253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033138257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-08T02:31:18,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033138454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033138454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033138454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033138454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033138462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,649 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/2e69b71c338146d9851bac602f05a657 2024-11-08T02:31:18,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/81da21565d1f499496eafc800d75f439 is 50, key is test_row_0/B:col10/1731033076022/Put/seqid=0 2024-11-08T02:31:18,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742009_1185 (size=12151) 2024-11-08T02:31:18,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033138757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033138758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033138759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033138759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033138765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-08T02:31:19,059 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/81da21565d1f499496eafc800d75f439 2024-11-08T02:31:19,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/311fda00f1344f658121a1e185b0dec5 is 50, key is test_row_0/C:col10/1731033076022/Put/seqid=0 2024-11-08T02:31:19,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742010_1186 (size=12151) 2024-11-08T02:31:19,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033139260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:19,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033139261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:19,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033139266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:19,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:19,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033139266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:19,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033139272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:19,495 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/311fda00f1344f658121a1e185b0dec5 2024-11-08T02:31:19,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/2e69b71c338146d9851bac602f05a657 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/2e69b71c338146d9851bac602f05a657 2024-11-08T02:31:19,503 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/2e69b71c338146d9851bac602f05a657, entries=150, sequenceid=235, filesize=30.4 K 2024-11-08T02:31:19,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/81da21565d1f499496eafc800d75f439 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/81da21565d1f499496eafc800d75f439 2024-11-08T02:31:19,511 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/81da21565d1f499496eafc800d75f439, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:31:19,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/311fda00f1344f658121a1e185b0dec5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/311fda00f1344f658121a1e185b0dec5 2024-11-08T02:31:19,515 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/311fda00f1344f658121a1e185b0dec5, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T02:31:19,516 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8b6bdcd9b78ee6cd044820b0655675ec in 1686ms, sequenceid=235, compaction requested=true 2024-11-08T02:31:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:19,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=56 2024-11-08T02:31:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=56 2024-11-08T02:31:19,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-08T02:31:19,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8380 sec 2024-11-08T02:31:19,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees in 1.8420 sec 2024-11-08T02:31:19,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-08T02:31:19,781 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 55 completed 2024-11-08T02:31:19,783 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees 2024-11-08T02:31:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T02:31:19,784 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=57, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:19,784 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=57, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:19,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T02:31:19,935 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:19,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-11-08T02:31:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:19,936 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:31:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:19,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:19,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:19,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:19,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:19,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:19,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d9425fa8a9fe4a92b0967b09470a425e_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033078149/Put/seqid=0 2024-11-08T02:31:19,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742011_1187 (size=12304) 2024-11-08T02:31:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T02:31:20,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:20,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:20,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033140278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033140278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033140278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033140279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033140283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:20,352 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d9425fa8a9fe4a92b0967b09470a425e_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d9425fa8a9fe4a92b0967b09470a425e_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:20,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9d66073d54ee4166a1b29635a1a813e1, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:20,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9d66073d54ee4166a1b29635a1a813e1 is 175, key is test_row_0/A:col10/1731033078149/Put/seqid=0 2024-11-08T02:31:20,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742012_1188 (size=31105) 2024-11-08T02:31:20,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033140384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033140384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033140384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T02:31:20,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033140388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033140586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033140586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033140586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033140590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,763 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9d66073d54ee4166a1b29635a1a813e1 2024-11-08T02:31:20,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/0f28ba3179db40ee8677877a383ce758 is 50, key is test_row_0/B:col10/1731033078149/Put/seqid=0 2024-11-08T02:31:20,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742013_1189 (size=12151) 2024-11-08T02:31:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T02:31:20,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033140888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033140889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033140890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033140895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:21,175 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/0f28ba3179db40ee8677877a383ce758 2024-11-08T02:31:21,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b9150a2e5f9e4fbd8289d59bdbb5db74 is 50, key is test_row_0/C:col10/1731033078149/Put/seqid=0 2024-11-08T02:31:21,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742014_1190 (size=12151) 2024-11-08T02:31:21,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033141392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:21,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033141394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:21,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033141395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:21,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033141403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:21,586 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b9150a2e5f9e4fbd8289d59bdbb5db74 2024-11-08T02:31:21,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9d66073d54ee4166a1b29635a1a813e1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9d66073d54ee4166a1b29635a1a813e1 2024-11-08T02:31:21,594 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9d66073d54ee4166a1b29635a1a813e1, entries=150, sequenceid=255, filesize=30.4 K 2024-11-08T02:31:21,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/0f28ba3179db40ee8677877a383ce758 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/0f28ba3179db40ee8677877a383ce758 2024-11-08T02:31:21,599 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/0f28ba3179db40ee8677877a383ce758, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T02:31:21,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/b9150a2e5f9e4fbd8289d59bdbb5db74 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b9150a2e5f9e4fbd8289d59bdbb5db74 2024-11-08T02:31:21,603 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b9150a2e5f9e4fbd8289d59bdbb5db74, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T02:31:21,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 8b6bdcd9b78ee6cd044820b0655675ec in 1668ms, sequenceid=255, compaction requested=true 2024-11-08T02:31:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-11-08T02:31:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=58 2024-11-08T02:31:21,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T02:31:21,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8210 sec 2024-11-08T02:31:21,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees in 1.8230 sec 2024-11-08T02:31:21,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-08T02:31:21,887 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 57 completed 2024-11-08T02:31:21,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:21,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees 2024-11-08T02:31:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-08T02:31:21,889 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=59, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:21,889 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=59, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:21,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-08T02:31:22,040 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-08T02:31:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:22,041 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:31:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:22,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:22,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086c4ad06fceaa4bb4810e8e1203263ea7_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033080277/Put/seqid=0 2024-11-08T02:31:22,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742015_1191 (size=12454) 2024-11-08T02:31:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-08T02:31:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:22,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033142334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033142400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033142400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033142405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033142410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033142446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:22,454 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086c4ad06fceaa4bb4810e8e1203263ea7_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086c4ad06fceaa4bb4810e8e1203263ea7_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/8fe5f94b17ad404b8eb850ee956e7ca7, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:22,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/8fe5f94b17ad404b8eb850ee956e7ca7 is 175, key is test_row_0/A:col10/1731033080277/Put/seqid=0 2024-11-08T02:31:22,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742016_1192 (size=31255) 2024-11-08T02:31:22,467 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/8fe5f94b17ad404b8eb850ee956e7ca7 2024-11-08T02:31:22,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/aa9e011faec84f11939a8b82d60feff8 is 50, key is test_row_0/B:col10/1731033080277/Put/seqid=0 2024-11-08T02:31:22,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742017_1193 (size=12301) 2024-11-08T02:31:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-08T02:31:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033142648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,887 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/aa9e011faec84f11939a8b82d60feff8 2024-11-08T02:31:22,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/0c3b0d0400c74ab7b552e0e185656a21 is 50, key is test_row_0/C:col10/1731033080277/Put/seqid=0 2024-11-08T02:31:22,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742018_1194 (size=12301) 2024-11-08T02:31:22,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033142950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-08T02:31:23,300 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/0c3b0d0400c74ab7b552e0e185656a21 2024-11-08T02:31:23,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/8fe5f94b17ad404b8eb850ee956e7ca7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/8fe5f94b17ad404b8eb850ee956e7ca7 2024-11-08T02:31:23,308 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/8fe5f94b17ad404b8eb850ee956e7ca7, entries=150, sequenceid=271, filesize=30.5 K 2024-11-08T02:31:23,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/aa9e011faec84f11939a8b82d60feff8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/aa9e011faec84f11939a8b82d60feff8 2024-11-08T02:31:23,313 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/aa9e011faec84f11939a8b82d60feff8, entries=150, sequenceid=271, filesize=12.0 K 2024-11-08T02:31:23,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/0c3b0d0400c74ab7b552e0e185656a21 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/0c3b0d0400c74ab7b552e0e185656a21 2024-11-08T02:31:23,320 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/0c3b0d0400c74ab7b552e0e185656a21, entries=150, sequenceid=271, filesize=12.0 K 2024-11-08T02:31:23,321 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8b6bdcd9b78ee6cd044820b0655675ec in 1279ms, sequenceid=271, compaction requested=true 2024-11-08T02:31:23,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:23,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:23,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-11-08T02:31:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-11-08T02:31:23,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-11-08T02:31:23,324 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-08T02:31:23,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees in 1.4360 sec 2024-11-08T02:31:23,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:23,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:31:23,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:23,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:23,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:23,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:23,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:23,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:23,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fe6df8137f5a4ba48f4906a69974589e_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033082329/Put/seqid=0 2024-11-08T02:31:23,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742019_1195 (size=14994) 2024-11-08T02:31:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033143502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:23,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:23,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033143608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:23,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033143811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:23,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:23,875 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fe6df8137f5a4ba48f4906a69974589e_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fe6df8137f5a4ba48f4906a69974589e_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:23,876 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a7255b6f98ea4e8488d51da926c6a433, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:23,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a7255b6f98ea4e8488d51da926c6a433 is 175, key is test_row_0/A:col10/1731033082329/Put/seqid=0 2024-11-08T02:31:23,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742020_1196 (size=39949) 2024-11-08T02:31:23,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-08T02:31:23,992 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 59 completed 2024-11-08T02:31:23,994 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees 2024-11-08T02:31:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-08T02:31:23,995 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=61, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:23,995 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=61, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:23,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:24,037 DEBUG [Thread-624 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3b727d6e to 127.0.0.1:56877 2024-11-08T02:31:24,037 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:24,039 DEBUG [Thread-622 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22e911df to 127.0.0.1:56877 2024-11-08T02:31:24,039 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:24,041 DEBUG [Thread-626 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1c7940d9 to 127.0.0.1:56877 2024-11-08T02:31:24,041 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:24,043 DEBUG [Thread-628 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2c38ee58 to 127.0.0.1:56877 2024-11-08T02:31:24,043 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-08T02:31:24,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:24,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033144115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,147 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:24,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:24,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a7255b6f98ea4e8488d51da926c6a433 2024-11-08T02:31:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-08T02:31:24,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/1342550ec489407d8ef63049a782df31 is 50, key is test_row_0/B:col10/1731033082329/Put/seqid=0 2024-11-08T02:31:24,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742021_1197 (size=12301) 2024-11-08T02:31:24,303 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:24,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:24,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:24,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46486 deadline: 1731033144417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46504 deadline: 1731033144417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,419 DEBUG [Thread-619 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4140 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:31:24,419 DEBUG [Thread-617 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4140 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:31:24,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46506 deadline: 1731033144421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,423 DEBUG [Thread-611 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4143 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:31:24,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46552 deadline: 1731033144427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,427 DEBUG [Thread-613 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4144 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:31:24,456 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:24,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:24,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-08T02:31:24,610 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:24,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46532 deadline: 1731033144622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/1342550ec489407d8ef63049a782df31 2024-11-08T02:31:24,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/cc73b03a144f406995519c922fd5c9a6 is 50, key is test_row_0/C:col10/1731033082329/Put/seqid=0 2024-11-08T02:31:24,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742022_1198 (size=12301) 2024-11-08T02:31:24,767 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,921 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:24,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:24,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:24,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:25,057 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:31:25,077 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:25,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:25,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:25,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:25,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:25,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:25,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-08T02:31:25,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/cc73b03a144f406995519c922fd5c9a6 2024-11-08T02:31:25,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/a7255b6f98ea4e8488d51da926c6a433 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a7255b6f98ea4e8488d51da926c6a433 2024-11-08T02:31:25,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a7255b6f98ea4e8488d51da926c6a433, entries=200, sequenceid=292, filesize=39.0 K 2024-11-08T02:31:25,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/1342550ec489407d8ef63049a782df31 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/1342550ec489407d8ef63049a782df31 2024-11-08T02:31:25,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/1342550ec489407d8ef63049a782df31, entries=150, sequenceid=292, filesize=12.0 K 2024-11-08T02:31:25,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/cc73b03a144f406995519c922fd5c9a6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/cc73b03a144f406995519c922fd5c9a6 2024-11-08T02:31:25,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/cc73b03a144f406995519c922fd5c9a6, entries=150, sequenceid=292, filesize=12.0 K 2024-11-08T02:31:25,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 8b6bdcd9b78ee6cd044820b0655675ec in 1683ms, sequenceid=292, compaction requested=true 2024-11-08T02:31:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:25,142 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-08T02:31:25,142 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-08T02:31:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8b6bdcd9b78ee6cd044820b0655675ec:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:25,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:25,145 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85767 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-08T02:31:25,145 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 227139 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-08T02:31:25,145 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/B is initiating minor compaction (all files) 2024-11-08T02:31:25,145 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/A is initiating minor compaction (all files) 2024-11-08T02:31:25,145 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/B in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:25,145 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/A in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:25,145 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e3be797d5ae94c5c994899ce330dc423, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/9b6356d8ee4841e8a8b2f9a685ac1da8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d4b116283743f6b3ef407ea0840a03, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/81da21565d1f499496eafc800d75f439, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/0f28ba3179db40ee8677877a383ce758, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/aa9e011faec84f11939a8b82d60feff8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/1342550ec489407d8ef63049a782df31] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=83.8 K 2024-11-08T02:31:25,145 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/6d9a3fcb2c3b4991be31bdf4866548ef, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/1d649b74253d47dd95a83d179728a7e9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/28111a23fc7544c19d74f1a1f86077b1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/2e69b71c338146d9851bac602f05a657, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9d66073d54ee4166a1b29635a1a813e1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/8fe5f94b17ad404b8eb850ee956e7ca7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a7255b6f98ea4e8488d51da926c6a433] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=221.8 K 2024-11-08T02:31:25,145 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:25,145 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/6d9a3fcb2c3b4991be31bdf4866548ef, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/1d649b74253d47dd95a83d179728a7e9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/28111a23fc7544c19d74f1a1f86077b1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/2e69b71c338146d9851bac602f05a657, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9d66073d54ee4166a1b29635a1a813e1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/8fe5f94b17ad404b8eb850ee956e7ca7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a7255b6f98ea4e8488d51da926c6a433] 2024-11-08T02:31:25,145 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e3be797d5ae94c5c994899ce330dc423, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731033071595 2024-11-08T02:31:25,145 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6d9a3fcb2c3b4991be31bdf4866548ef, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731033071595 2024-11-08T02:31:25,146 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 9b6356d8ee4841e8a8b2f9a685ac1da8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731033071710 2024-11-08T02:31:25,146 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1d649b74253d47dd95a83d179728a7e9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731033071710 2024-11-08T02:31:25,146 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e4d4b116283743f6b3ef407ea0840a03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731033073851 2024-11-08T02:31:25,146 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 28111a23fc7544c19d74f1a1f86077b1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731033073851 2024-11-08T02:31:25,146 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 81da21565d1f499496eafc800d75f439, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731033076021 2024-11-08T02:31:25,146 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2e69b71c338146d9851bac602f05a657, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731033076021 2024-11-08T02:31:25,146 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 0f28ba3179db40ee8677877a383ce758, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731033078147 2024-11-08T02:31:25,147 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9d66073d54ee4166a1b29635a1a813e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731033078147 2024-11-08T02:31:25,147 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting aa9e011faec84f11939a8b82d60feff8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731033080276 2024-11-08T02:31:25,147 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8fe5f94b17ad404b8eb850ee956e7ca7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731033080276 2024-11-08T02:31:25,147 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1342550ec489407d8ef63049a782df31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731033082319 2024-11-08T02:31:25,147 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting a7255b6f98ea4e8488d51da926c6a433, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731033082319 2024-11-08T02:31:25,160 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:25,167 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411086c99680be10b4ba7bea165f3dd276b38_8b6bdcd9b78ee6cd044820b0655675ec store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:25,167 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#B#compaction#162 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:25,167 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/2c328af7ca97432da4daf354bcf64c9a is 50, key is test_row_0/B:col10/1731033082329/Put/seqid=0 2024-11-08T02:31:25,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742023_1199 (size=12949) 2024-11-08T02:31:25,212 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411086c99680be10b4ba7bea165f3dd276b38_8b6bdcd9b78ee6cd044820b0655675ec, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:25,212 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086c99680be10b4ba7bea165f3dd276b38_8b6bdcd9b78ee6cd044820b0655675ec because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:25,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742024_1200 (size=4469) 2024-11-08T02:31:25,233 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:25,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-08T02:31:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:25,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:31:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:25,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:25,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:25,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082d985224c4af4b68a9110fd8aa720c46_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_0/A:col10/1731033083496/Put/seqid=0 2024-11-08T02:31:25,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742025_1201 (size=12454) 2024-11-08T02:31:25,587 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/2c328af7ca97432da4daf354bcf64c9a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/2c328af7ca97432da4daf354bcf64c9a 2024-11-08T02:31:25,594 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 7 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/B of 8b6bdcd9b78ee6cd044820b0655675ec into 2c328af7ca97432da4daf354bcf64c9a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:25,594 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:25,595 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/B, priority=9, startTime=1731033085142; duration=0sec 2024-11-08T02:31:25,595 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:25,595 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:B 2024-11-08T02:31:25,595 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-08T02:31:25,598 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 85767 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-08T02:31:25,598 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 8b6bdcd9b78ee6cd044820b0655675ec/C is initiating minor compaction (all files) 2024-11-08T02:31:25,598 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 8b6bdcd9b78ee6cd044820b0655675ec/C in TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:25,598 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b6ec26b2d5bc4e3287e75ef84d17ded9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9c235f507fcc4b08b7118089401cb41f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9d537a8ad62d4f47b10540d0e9c88893, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/311fda00f1344f658121a1e185b0dec5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b9150a2e5f9e4fbd8289d59bdbb5db74, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/0c3b0d0400c74ab7b552e0e185656a21, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/cc73b03a144f406995519c922fd5c9a6] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp, totalSize=83.8 K 2024-11-08T02:31:25,599 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b6ec26b2d5bc4e3287e75ef84d17ded9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731033071595 2024-11-08T02:31:25,599 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 9c235f507fcc4b08b7118089401cb41f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731033071710 2024-11-08T02:31:25,600 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 9d537a8ad62d4f47b10540d0e9c88893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731033073851 2024-11-08T02:31:25,600 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 311fda00f1344f658121a1e185b0dec5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731033076021 2024-11-08T02:31:25,601 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b9150a2e5f9e4fbd8289d59bdbb5db74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731033078147 2024-11-08T02:31:25,601 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 0c3b0d0400c74ab7b552e0e185656a21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731033080276 2024-11-08T02:31:25,602 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting cc73b03a144f406995519c922fd5c9a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731033082319 2024-11-08T02:31:25,617 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#A#compaction#163 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:25,617 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/09f38d0aea6b47229c861c02c23168e8 is 175, key is test_row_0/A:col10/1731033082329/Put/seqid=0 2024-11-08T02:31:25,618 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8b6bdcd9b78ee6cd044820b0655675ec#C#compaction#165 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:25,619 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/c8d33bc573484413b6c4aff286e9ecbc is 50, key is test_row_0/C:col10/1731033082329/Put/seqid=0 2024-11-08T02:31:25,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742026_1202 (size=31903) 2024-11-08T02:31:25,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742027_1203 (size=12949) 2024-11-08T02:31:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. as already flushing 2024-11-08T02:31:25,626 DEBUG [Thread-615 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x454f1431 to 127.0.0.1:56877 2024-11-08T02:31:25,626 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:25,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:25,656 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082d985224c4af4b68a9110fd8aa720c46_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082d985224c4af4b68a9110fd8aa720c46_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:25,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/15ece73cfda241279a84a8606acfb149, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/15ece73cfda241279a84a8606acfb149 is 175, key is test_row_0/A:col10/1731033083496/Put/seqid=0 2024-11-08T02:31:25,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742028_1204 (size=31255) 2024-11-08T02:31:26,033 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/09f38d0aea6b47229c861c02c23168e8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/09f38d0aea6b47229c861c02c23168e8 2024-11-08T02:31:26,034 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/c8d33bc573484413b6c4aff286e9ecbc as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c8d33bc573484413b6c4aff286e9ecbc 2024-11-08T02:31:26,040 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 7 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/A of 8b6bdcd9b78ee6cd044820b0655675ec into 09f38d0aea6b47229c861c02c23168e8(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:26,040 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:26,040 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/A, priority=9, startTime=1731033085142; duration=0sec 2024-11-08T02:31:26,040 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:26,040 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:A 2024-11-08T02:31:26,040 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 7 (all) file(s) in 8b6bdcd9b78ee6cd044820b0655675ec/C of 8b6bdcd9b78ee6cd044820b0655675ec into c8d33bc573484413b6c4aff286e9ecbc(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:26,040 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:26,040 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec., storeName=8b6bdcd9b78ee6cd044820b0655675ec/C, priority=9, startTime=1731033085142; duration=0sec 2024-11-08T02:31:26,040 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:26,040 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8b6bdcd9b78ee6cd044820b0655675ec:C 2024-11-08T02:31:26,064 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=307, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/15ece73cfda241279a84a8606acfb149 2024-11-08T02:31:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/4ae155e0eec24db3883ce772b47365a0 is 50, key is test_row_0/B:col10/1731033083496/Put/seqid=0 2024-11-08T02:31:26,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742029_1205 (size=12301) 2024-11-08T02:31:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-08T02:31:26,475 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/4ae155e0eec24db3883ce772b47365a0 2024-11-08T02:31:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/42721d2479d74e89aef3de51b2040149 is 50, key is test_row_0/C:col10/1731033083496/Put/seqid=0 2024-11-08T02:31:26,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742030_1206 (size=12301) 2024-11-08T02:31:26,888 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/42721d2479d74e89aef3de51b2040149 2024-11-08T02:31:26,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/15ece73cfda241279a84a8606acfb149 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/15ece73cfda241279a84a8606acfb149 2024-11-08T02:31:26,903 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/15ece73cfda241279a84a8606acfb149, entries=150, sequenceid=307, filesize=30.5 K 2024-11-08T02:31:26,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/4ae155e0eec24db3883ce772b47365a0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/4ae155e0eec24db3883ce772b47365a0 2024-11-08T02:31:26,907 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/4ae155e0eec24db3883ce772b47365a0, entries=150, sequenceid=307, filesize=12.0 K 2024-11-08T02:31:26,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/42721d2479d74e89aef3de51b2040149 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/42721d2479d74e89aef3de51b2040149 2024-11-08T02:31:26,911 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/42721d2479d74e89aef3de51b2040149, entries=150, sequenceid=307, filesize=12.0 K 2024-11-08T02:31:26,911 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=6.71 KB/6870 for 8b6bdcd9b78ee6cd044820b0655675ec in 1677ms, sequenceid=307, compaction requested=false 2024-11-08T02:31:26,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2538): Flush status journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:26,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:26,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-08T02:31:26,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-11-08T02:31:26,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-08T02:31:26,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9170 sec 2024-11-08T02:31:26,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees in 2.9190 sec 2024-11-08T02:31:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-08T02:31:28,103 INFO [Thread-621 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 61 completed 2024-11-08T02:31:28,433 DEBUG [Thread-611 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x79982672 to 127.0.0.1:56877 2024-11-08T02:31:28,433 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:28,442 DEBUG [Thread-619 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x367f47f7 to 127.0.0.1:56877 2024-11-08T02:31:28,442 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:28,461 DEBUG [Thread-617 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x505d5ccd to 127.0.0.1:56877 2024-11-08T02:31:28,461 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:28,461 DEBUG [Thread-613 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7b4bd1ba to 127.0.0.1:56877 2024-11-08T02:31:28,461 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:28,461 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:31:28,461 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-08T02:31:28,461 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-08T02:31:28,461 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-08T02:31:28,461 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-08T02:31:28,461 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-08T02:31:28,461 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:31:28,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5854 2024-11-08T02:31:28,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5795 2024-11-08T02:31:28,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:31:28,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2463 2024-11-08T02:31:28,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7388 rows 2024-11-08T02:31:28,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2454 2024-11-08T02:31:28,462 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7360 rows 2024-11-08T02:31:28,462 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T02:31:28,462 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x118b007e to 127.0.0.1:56877 2024-11-08T02:31:28,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:28,464 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-08T02:31:28,464 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:31:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T02:31:28,469 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033088468"}]},"ts":"1731033088468"} 2024-11-08T02:31:28,470 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:31:28,486 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:31:28,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:31:28,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, UNASSIGN}] 2024-11-08T02:31:28,488 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=65, ppid=64, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, UNASSIGN 2024-11-08T02:31:28,489 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=65 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:28,490 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:31:28,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; CloseRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:31:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T02:31:28,642 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:28,642 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] handler.UnassignRegionHandler(124): Close 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:28,643 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:31:28,643 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1681): Closing 8b6bdcd9b78ee6cd044820b0655675ec, disabling compactions & flushes 2024-11-08T02:31:28,643 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:28,643 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:28,643 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. after waiting 0 ms 2024-11-08T02:31:28,643 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:28,643 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(2837): Flushing 8b6bdcd9b78ee6cd044820b0655675ec 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T02:31:28,644 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=A 2024-11-08T02:31:28,644 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:28,644 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=B 2024-11-08T02:31:28,644 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:28,644 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8b6bdcd9b78ee6cd044820b0655675ec, store=C 2024-11-08T02:31:28,644 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:28,656 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110876ec1758f53b4d7e9b2e217579587119_8b6bdcd9b78ee6cd044820b0655675ec is 50, key is test_row_1/A:col10/1731033088459/Put/seqid=0 2024-11-08T02:31:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742031_1207 (size=9914) 2024-11-08T02:31:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T02:31:29,061 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:29,071 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110876ec1758f53b4d7e9b2e217579587119_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110876ec1758f53b4d7e9b2e217579587119_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T02:31:29,073 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9a4c56791f3f483ca5e166d16b8651a3, store: [table=TestAcidGuarantees family=A region=8b6bdcd9b78ee6cd044820b0655675ec] 2024-11-08T02:31:29,074 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9a4c56791f3f483ca5e166d16b8651a3 is 175, key is test_row_1/A:col10/1731033088459/Put/seqid=0 2024-11-08T02:31:29,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742032_1208 (size=22561) 2024-11-08T02:31:29,479 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9a4c56791f3f483ca5e166d16b8651a3 2024-11-08T02:31:29,492 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b2f388fd8c3046daadebef0cbd931171 is 50, key is test_row_1/B:col10/1731033088459/Put/seqid=0 2024-11-08T02:31:29,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742033_1209 (size=9857) 2024-11-08T02:31:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T02:31:29,898 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b2f388fd8c3046daadebef0cbd931171 2024-11-08T02:31:29,901 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46319,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:29,913 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/2d1ebae76d404e0fb200c58036d22cae is 50, key is test_row_1/C:col10/1731033088459/Put/seqid=0 2024-11-08T02:31:29,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742034_1210 (size=9857) 2024-11-08T02:31:30,320 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/2d1ebae76d404e0fb200c58036d22cae 2024-11-08T02:31:30,329 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/A/9a4c56791f3f483ca5e166d16b8651a3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9a4c56791f3f483ca5e166d16b8651a3 2024-11-08T02:31:30,336 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9a4c56791f3f483ca5e166d16b8651a3, entries=100, sequenceid=318, filesize=22.0 K 2024-11-08T02:31:30,337 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/B/b2f388fd8c3046daadebef0cbd931171 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b2f388fd8c3046daadebef0cbd931171 2024-11-08T02:31:30,341 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b2f388fd8c3046daadebef0cbd931171, entries=100, sequenceid=318, filesize=9.6 K 2024-11-08T02:31:30,342 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/.tmp/C/2d1ebae76d404e0fb200c58036d22cae as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/2d1ebae76d404e0fb200c58036d22cae 2024-11-08T02:31:30,346 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/2d1ebae76d404e0fb200c58036d22cae, entries=100, sequenceid=318, filesize=9.6 K 2024-11-08T02:31:30,347 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 8b6bdcd9b78ee6cd044820b0655675ec in 1704ms, sequenceid=318, compaction requested=true 2024-11-08T02:31:30,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e228c787be3e4d43ae63ef33d3a559e8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6c3f90ba12f4feb8bf299d56a3a33ba, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/b420f3672b4b40fc8e1e2a35be5ddd33, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/83ef3682c1fd4748905f0cdb8ad3f71d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/cf8e74a2c0d54cb1a019c2f7a59503a1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f24a068dd2974cb7b10cdeae41ab3545, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e9b70d4a6fc747f1a8426fc0547f382e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6ecc0de2fdc4f2b852b9cb5766e1b78, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d0e3c127f3674745b9fd77f73e6a5b88, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f8697e5b4d43437c82123a7c53cb5609, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d019db358c4b44c2acc0c3e7a0ae0d6d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/68996fa3133a4398bc037091828c5ef9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/6d9a3fcb2c3b4991be31bdf4866548ef, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/1d649b74253d47dd95a83d179728a7e9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/28111a23fc7544c19d74f1a1f86077b1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/2e69b71c338146d9851bac602f05a657, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9d66073d54ee4166a1b29635a1a813e1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/8fe5f94b17ad404b8eb850ee956e7ca7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a7255b6f98ea4e8488d51da926c6a433] to archive 2024-11-08T02:31:30,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:31:30,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e228c787be3e4d43ae63ef33d3a559e8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e228c787be3e4d43ae63ef33d3a559e8 2024-11-08T02:31:30,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6c3f90ba12f4feb8bf299d56a3a33ba to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6c3f90ba12f4feb8bf299d56a3a33ba 2024-11-08T02:31:30,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/b420f3672b4b40fc8e1e2a35be5ddd33 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/b420f3672b4b40fc8e1e2a35be5ddd33 2024-11-08T02:31:30,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/83ef3682c1fd4748905f0cdb8ad3f71d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/83ef3682c1fd4748905f0cdb8ad3f71d 2024-11-08T02:31:30,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/cf8e74a2c0d54cb1a019c2f7a59503a1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/cf8e74a2c0d54cb1a019c2f7a59503a1 2024-11-08T02:31:30,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f24a068dd2974cb7b10cdeae41ab3545 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f24a068dd2974cb7b10cdeae41ab3545 2024-11-08T02:31:30,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e9b70d4a6fc747f1a8426fc0547f382e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/e9b70d4a6fc747f1a8426fc0547f382e 2024-11-08T02:31:30,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6ecc0de2fdc4f2b852b9cb5766e1b78 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a6ecc0de2fdc4f2b852b9cb5766e1b78 2024-11-08T02:31:30,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d0e3c127f3674745b9fd77f73e6a5b88 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d0e3c127f3674745b9fd77f73e6a5b88 2024-11-08T02:31:30,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f8697e5b4d43437c82123a7c53cb5609 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/f8697e5b4d43437c82123a7c53cb5609 2024-11-08T02:31:30,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d019db358c4b44c2acc0c3e7a0ae0d6d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/d019db358c4b44c2acc0c3e7a0ae0d6d 2024-11-08T02:31:30,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/68996fa3133a4398bc037091828c5ef9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/68996fa3133a4398bc037091828c5ef9 2024-11-08T02:31:30,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/6d9a3fcb2c3b4991be31bdf4866548ef to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/6d9a3fcb2c3b4991be31bdf4866548ef 2024-11-08T02:31:30,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/1d649b74253d47dd95a83d179728a7e9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/1d649b74253d47dd95a83d179728a7e9 2024-11-08T02:31:30,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/28111a23fc7544c19d74f1a1f86077b1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/28111a23fc7544c19d74f1a1f86077b1 2024-11-08T02:31:30,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/2e69b71c338146d9851bac602f05a657 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/2e69b71c338146d9851bac602f05a657 2024-11-08T02:31:30,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9d66073d54ee4166a1b29635a1a813e1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9d66073d54ee4166a1b29635a1a813e1 2024-11-08T02:31:30,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/8fe5f94b17ad404b8eb850ee956e7ca7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/8fe5f94b17ad404b8eb850ee956e7ca7 2024-11-08T02:31:30,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a7255b6f98ea4e8488d51da926c6a433 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/a7255b6f98ea4e8488d51da926c6a433 2024-11-08T02:31:30,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d22ac4bde941eb8eab802595750aa1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/8f291cab6af04cbe90aef95b41c8d398, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/44515627f9934dbab055261d9dbaacdf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/974c8b300d004c50a4c32bb79a67a734, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5d0354565eb749f0be4dbd6f723b092c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/a90d31aea9564b2d9562e180fb113689, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/c020daa3856e468fbc301d79feeaf760, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b414012759a642a58e7d69329c90e019, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/6531c072c7874d21870d7943738adfea, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5199fa2b7a2e4dfda973627fad748ef8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b968cf08fb74431099867d2255704adb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e3be797d5ae94c5c994899ce330dc423, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b61cb80d384c44cf9c25d318a37b7974, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/9b6356d8ee4841e8a8b2f9a685ac1da8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d4b116283743f6b3ef407ea0840a03, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/81da21565d1f499496eafc800d75f439, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/0f28ba3179db40ee8677877a383ce758, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/aa9e011faec84f11939a8b82d60feff8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/1342550ec489407d8ef63049a782df31] to archive 2024-11-08T02:31:30,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:31:30,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d22ac4bde941eb8eab802595750aa1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d22ac4bde941eb8eab802595750aa1 2024-11-08T02:31:30,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/8f291cab6af04cbe90aef95b41c8d398 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/8f291cab6af04cbe90aef95b41c8d398 2024-11-08T02:31:30,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/44515627f9934dbab055261d9dbaacdf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/44515627f9934dbab055261d9dbaacdf 2024-11-08T02:31:30,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/974c8b300d004c50a4c32bb79a67a734 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/974c8b300d004c50a4c32bb79a67a734 2024-11-08T02:31:30,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5d0354565eb749f0be4dbd6f723b092c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5d0354565eb749f0be4dbd6f723b092c 2024-11-08T02:31:30,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/a90d31aea9564b2d9562e180fb113689 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/a90d31aea9564b2d9562e180fb113689 2024-11-08T02:31:30,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/c020daa3856e468fbc301d79feeaf760 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/c020daa3856e468fbc301d79feeaf760 2024-11-08T02:31:30,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b414012759a642a58e7d69329c90e019 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b414012759a642a58e7d69329c90e019 2024-11-08T02:31:30,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/6531c072c7874d21870d7943738adfea to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/6531c072c7874d21870d7943738adfea 2024-11-08T02:31:30,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5199fa2b7a2e4dfda973627fad748ef8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/5199fa2b7a2e4dfda973627fad748ef8 2024-11-08T02:31:30,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b968cf08fb74431099867d2255704adb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b968cf08fb74431099867d2255704adb 2024-11-08T02:31:30,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e3be797d5ae94c5c994899ce330dc423 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e3be797d5ae94c5c994899ce330dc423 2024-11-08T02:31:30,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b61cb80d384c44cf9c25d318a37b7974 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b61cb80d384c44cf9c25d318a37b7974 2024-11-08T02:31:30,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/9b6356d8ee4841e8a8b2f9a685ac1da8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/9b6356d8ee4841e8a8b2f9a685ac1da8 2024-11-08T02:31:30,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d4b116283743f6b3ef407ea0840a03 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/e4d4b116283743f6b3ef407ea0840a03 2024-11-08T02:31:30,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/81da21565d1f499496eafc800d75f439 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/81da21565d1f499496eafc800d75f439 2024-11-08T02:31:30,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/0f28ba3179db40ee8677877a383ce758 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/0f28ba3179db40ee8677877a383ce758 2024-11-08T02:31:30,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/aa9e011faec84f11939a8b82d60feff8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/aa9e011faec84f11939a8b82d60feff8 2024-11-08T02:31:30,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/1342550ec489407d8ef63049a782df31 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/1342550ec489407d8ef63049a782df31 2024-11-08T02:31:30,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/e608f68f70544d4fa12b8ff498526a34, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fc803009282e42478ab19dc7af7d75d3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/aa72908f197a47d7b28cd5ceaf65eec8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/1fdfb1c0498e4ebead1a495f67665a89, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/932dee5655b440baaee7312d6875d323, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4254184686448ed80ef095ca8d83cb6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4ec00cc7feb469a9746ca609274daed, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/436515431bfa4640a238bd8c18b4e9b3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fea5c0fe49ec4d47b9d2441f5af1e73d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c1718b581c2745259935085314abbf9a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/f6ceac57e6c44002a628ded3a2fcfd0d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b6ec26b2d5bc4e3287e75ef84d17ded9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/a1641cfe169a4f6cb3c46980e9fdaace, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9c235f507fcc4b08b7118089401cb41f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9d537a8ad62d4f47b10540d0e9c88893, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/311fda00f1344f658121a1e185b0dec5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b9150a2e5f9e4fbd8289d59bdbb5db74, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/0c3b0d0400c74ab7b552e0e185656a21, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/cc73b03a144f406995519c922fd5c9a6] to archive 2024-11-08T02:31:30,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:31:30,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/e608f68f70544d4fa12b8ff498526a34 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/e608f68f70544d4fa12b8ff498526a34 2024-11-08T02:31:30,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fc803009282e42478ab19dc7af7d75d3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fc803009282e42478ab19dc7af7d75d3 2024-11-08T02:31:30,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/aa72908f197a47d7b28cd5ceaf65eec8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/aa72908f197a47d7b28cd5ceaf65eec8 2024-11-08T02:31:30,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/1fdfb1c0498e4ebead1a495f67665a89 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/1fdfb1c0498e4ebead1a495f67665a89 2024-11-08T02:31:30,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/932dee5655b440baaee7312d6875d323 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/932dee5655b440baaee7312d6875d323 2024-11-08T02:31:30,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4254184686448ed80ef095ca8d83cb6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4254184686448ed80ef095ca8d83cb6 2024-11-08T02:31:30,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4ec00cc7feb469a9746ca609274daed to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b4ec00cc7feb469a9746ca609274daed 2024-11-08T02:31:30,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/436515431bfa4640a238bd8c18b4e9b3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/436515431bfa4640a238bd8c18b4e9b3 2024-11-08T02:31:30,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fea5c0fe49ec4d47b9d2441f5af1e73d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/fea5c0fe49ec4d47b9d2441f5af1e73d 2024-11-08T02:31:30,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c1718b581c2745259935085314abbf9a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c1718b581c2745259935085314abbf9a 2024-11-08T02:31:30,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/f6ceac57e6c44002a628ded3a2fcfd0d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/f6ceac57e6c44002a628ded3a2fcfd0d 2024-11-08T02:31:30,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b6ec26b2d5bc4e3287e75ef84d17ded9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b6ec26b2d5bc4e3287e75ef84d17ded9 2024-11-08T02:31:30,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/a1641cfe169a4f6cb3c46980e9fdaace to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/a1641cfe169a4f6cb3c46980e9fdaace 2024-11-08T02:31:30,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9c235f507fcc4b08b7118089401cb41f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9c235f507fcc4b08b7118089401cb41f 2024-11-08T02:31:30,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9d537a8ad62d4f47b10540d0e9c88893 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/9d537a8ad62d4f47b10540d0e9c88893 2024-11-08T02:31:30,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/311fda00f1344f658121a1e185b0dec5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/311fda00f1344f658121a1e185b0dec5 2024-11-08T02:31:30,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b9150a2e5f9e4fbd8289d59bdbb5db74 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/b9150a2e5f9e4fbd8289d59bdbb5db74 2024-11-08T02:31:30,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/0c3b0d0400c74ab7b552e0e185656a21 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/0c3b0d0400c74ab7b552e0e185656a21 2024-11-08T02:31:30,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/cc73b03a144f406995519c922fd5c9a6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/cc73b03a144f406995519c922fd5c9a6 2024-11-08T02:31:30,408 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/recovered.edits/321.seqid, newMaxSeqId=321, maxSeqId=4 2024-11-08T02:31:30,408 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec. 2024-11-08T02:31:30,408 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1635): Region close journal for 8b6bdcd9b78ee6cd044820b0655675ec: 2024-11-08T02:31:30,410 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] handler.UnassignRegionHandler(170): Closed 8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,410 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=65 updating hbase:meta row=8b6bdcd9b78ee6cd044820b0655675ec, regionState=CLOSED 2024-11-08T02:31:30,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-11-08T02:31:30,412 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; CloseRegionProcedure 8b6bdcd9b78ee6cd044820b0655675ec, server=ecdbff65922a,40577,1731033026611 in 1.9210 sec 2024-11-08T02:31:30,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T02:31:30,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=8b6bdcd9b78ee6cd044820b0655675ec, UNASSIGN in 1.9240 sec 2024-11-08T02:31:30,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-08T02:31:30,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-08T02:31:30,415 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033090414"}]},"ts":"1731033090414"} 2024-11-08T02:31:30,415 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:31:30,453 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:31:30,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9890 sec 2024-11-08T02:31:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-08T02:31:30,574 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 63 completed 2024-11-08T02:31:30,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:31:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:30,578 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=67, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:30,579 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=67, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-08T02:31:30,582 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,588 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/recovered.edits] 2024-11-08T02:31:30,594 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/09f38d0aea6b47229c861c02c23168e8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/09f38d0aea6b47229c861c02c23168e8 2024-11-08T02:31:30,597 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/15ece73cfda241279a84a8606acfb149 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/15ece73cfda241279a84a8606acfb149 2024-11-08T02:31:30,600 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9a4c56791f3f483ca5e166d16b8651a3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/A/9a4c56791f3f483ca5e166d16b8651a3 2024-11-08T02:31:30,605 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/2c328af7ca97432da4daf354bcf64c9a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/2c328af7ca97432da4daf354bcf64c9a 2024-11-08T02:31:30,607 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/4ae155e0eec24db3883ce772b47365a0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/4ae155e0eec24db3883ce772b47365a0 2024-11-08T02:31:30,610 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b2f388fd8c3046daadebef0cbd931171 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/B/b2f388fd8c3046daadebef0cbd931171 2024-11-08T02:31:30,612 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/2d1ebae76d404e0fb200c58036d22cae to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/2d1ebae76d404e0fb200c58036d22cae 2024-11-08T02:31:30,613 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/42721d2479d74e89aef3de51b2040149 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/42721d2479d74e89aef3de51b2040149 2024-11-08T02:31:30,614 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c8d33bc573484413b6c4aff286e9ecbc to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/C/c8d33bc573484413b6c4aff286e9ecbc 2024-11-08T02:31:30,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/recovered.edits/321.seqid to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec/recovered.edits/321.seqid 2024-11-08T02:31:30,617 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,618 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:31:30,618 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:31:30,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T02:31:30,622 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108225abf649759410da26729bc75ae1ad2_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108225abf649759410da26729bc75ae1ad2_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,623 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082307278ea2ab4340ad87b4d1bb1e023b_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082307278ea2ab4340ad87b4d1bb1e023b_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,624 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082d985224c4af4b68a9110fd8aa720c46_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082d985224c4af4b68a9110fd8aa720c46_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,625 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085fb425bc58d74422bdb20b85486ef569_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085fb425bc58d74422bdb20b85486ef569_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,626 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086c4ad06fceaa4bb4810e8e1203263ea7_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086c4ad06fceaa4bb4810e8e1203263ea7_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,627 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110873d8de4f6dd14c3281cb1627d463c97a_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110873d8de4f6dd14c3281cb1627d463c97a_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,628 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110876ec1758f53b4d7e9b2e217579587119_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110876ec1758f53b4d7e9b2e217579587119_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,629 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087b090aaf07b24bd48bdc6170e36f8f19_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411087b090aaf07b24bd48bdc6170e36f8f19_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,631 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a4e25fbfe5949fba90fd2d0c7edc0ae_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089a4e25fbfe5949fba90fd2d0c7edc0ae_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,632 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089af3d5a3326d40bcaf20a483e629d090_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411089af3d5a3326d40bcaf20a483e629d090_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,633 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bda8eecf7e3d45a493f703f6457b4437_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bda8eecf7e3d45a493f703f6457b4437_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,634 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d09d73ecae54474aaa9bb4383085c27d_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d09d73ecae54474aaa9bb4383085c27d_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,635 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d5b8c7d494264972932db1679209f84b_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d5b8c7d494264972932db1679209f84b_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,636 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d9425fa8a9fe4a92b0967b09470a425e_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d9425fa8a9fe4a92b0967b09470a425e_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,637 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eb36585c30194e3790a4f98c5a15e535_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eb36585c30194e3790a4f98c5a15e535_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,637 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ebdd6235d03545e7b187be96b21842c5_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ebdd6235d03545e7b187be96b21842c5_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,638 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fe6df8137f5a4ba48f4906a69974589e_8b6bdcd9b78ee6cd044820b0655675ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fe6df8137f5a4ba48f4906a69974589e_8b6bdcd9b78ee6cd044820b0655675ec 2024-11-08T02:31:30,639 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:31:30,641 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=67, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:30,642 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:31:30,644 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:31:30,645 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=67, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:30,645 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:31:30,645 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731033090645"}]},"ts":"9223372036854775807"} 2024-11-08T02:31:30,647 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-08T02:31:30,647 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 8b6bdcd9b78ee6cd044820b0655675ec, NAME => 'TestAcidGuarantees,,1731033060719.8b6bdcd9b78ee6cd044820b0655675ec.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:31:30,647 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:31:30,648 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731033090647"}]},"ts":"9223372036854775807"} 2024-11-08T02:31:30,649 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-08T02:31:30,658 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=67, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:30,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-08T02:31:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-08T02:31:30,680 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 67 completed 2024-11-08T02:31:30,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMobMixedAtomicity Thread=239 (was 239), OpenFileDescriptor=456 (was 458), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 238), ProcessCount=11 (was 11), AvailableMemoryMB=2223 (was 2318) 2024-11-08T02:31:30,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testGetAtomicity Thread=239, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=2223 2024-11-08T02:31:30,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:31:30,702 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:31:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=68, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:31:30,703 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:31:30,704 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:30,704 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 68 2024-11-08T02:31:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-08T02:31:30,704 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:31:30,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742035_1211 (size=960) 2024-11-08T02:31:30,729 ERROR [LeaseRenewer:jenkins@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46319,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-08T02:31:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-08T02:31:31,118 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:31:31,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742036_1212 (size=53) 2024-11-08T02:31:31,227 DEBUG [master/ecdbff65922a:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 934a9f648376cf09654e404de226250e changed from -1.0 to 0.0, refreshing cache 2024-11-08T02:31:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-08T02:31:31,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:31:31,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 0bc3c725c17ccfcde445d058bf2f374c, disabling compactions & flushes 2024-11-08T02:31:31,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,529 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. after waiting 0 ms 2024-11-08T02:31:31,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,530 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,530 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:31,532 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:31:31,533 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731033091532"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731033091532"}]},"ts":"1731033091532"} 2024-11-08T02:31:31,536 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T02:31:31,538 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:31:31,539 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033091538"}]},"ts":"1731033091538"} 2024-11-08T02:31:31,540 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:31:31,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bc3c725c17ccfcde445d058bf2f374c, ASSIGN}] 2024-11-08T02:31:31,596 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bc3c725c17ccfcde445d058bf2f374c, ASSIGN 2024-11-08T02:31:31,597 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bc3c725c17ccfcde445d058bf2f374c, ASSIGN; state=OFFLINE, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=false 2024-11-08T02:31:31,748 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=0bc3c725c17ccfcde445d058bf2f374c, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:31,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; OpenRegionProcedure 0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:31:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-08T02:31:31,906 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:31,914 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,914 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7285): Opening region: {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:31:31,915 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,915 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:31:31,916 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7327): checking encryption for 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,916 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7330): checking classloading for 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,918 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,920 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:31,920 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bc3c725c17ccfcde445d058bf2f374c columnFamilyName A 2024-11-08T02:31:31,920 DEBUG [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:31,921 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.HStore(327): Store=0bc3c725c17ccfcde445d058bf2f374c/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:31,921 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,922 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:31,923 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bc3c725c17ccfcde445d058bf2f374c columnFamilyName B 2024-11-08T02:31:31,923 DEBUG [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:31,923 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.HStore(327): Store=0bc3c725c17ccfcde445d058bf2f374c/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:31,924 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,925 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:31:31,925 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bc3c725c17ccfcde445d058bf2f374c columnFamilyName C 2024-11-08T02:31:31,925 DEBUG [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:31:31,926 INFO [StoreOpener-0bc3c725c17ccfcde445d058bf2f374c-1 {}] regionserver.HStore(327): Store=0bc3c725c17ccfcde445d058bf2f374c/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:31:31,926 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,927 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,927 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,929 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:31:31,931 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1085): writing seq id for 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:31,933 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:31:31,934 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1102): Opened 0bc3c725c17ccfcde445d058bf2f374c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69531511, jitterRate=0.03610025346279144}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:31:31,934 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1001): Region open journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:31,935 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., pid=70, masterSystemTime=1731033091905 2024-11-08T02:31:31,936 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,936 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:31,937 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=0bc3c725c17ccfcde445d058bf2f374c, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:31,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-08T02:31:31,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; OpenRegionProcedure 0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 in 187 msec 2024-11-08T02:31:31,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=69, resume processing ppid=68 2024-11-08T02:31:31,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bc3c725c17ccfcde445d058bf2f374c, ASSIGN in 345 msec 2024-11-08T02:31:31,941 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:31:31,941 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033091941"}]},"ts":"1731033091941"} 2024-11-08T02:31:31,941 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:31:31,949 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:31:31,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2470 sec 2024-11-08T02:31:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-08T02:31:32,813 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 68 completed 2024-11-08T02:31:32,817 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a569490 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c1ac389 2024-11-08T02:31:32,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44645c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,832 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36170, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:31:32,835 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33468, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:31:32,839 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6862e3ce to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28e73c0 2024-11-08T02:31:32,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ee0130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,851 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d296fed to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c480dfb 2024-11-08T02:31:32,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683b64c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,864 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08d0caa5 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34cb3991 2024-11-08T02:31:32,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e55eb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,878 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x43f04e0e to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e9ae050 2024-11-08T02:31:32,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a703d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,889 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x560ec309 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fef31f8 2024-11-08T02:31:32,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ed1e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,903 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0eb04aeb to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72537a47 2024-11-08T02:31:32,916 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88aa519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,919 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a0e9c8f to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36642cb 2024-11-08T02:31:32,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e998dd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,930 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d68f787 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c299cfb 2024-11-08T02:31:32,942 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4c79b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,944 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10e6bf6a to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@605827c9 2024-11-08T02:31:32,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1403c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,955 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1730a60f to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3677bd4f 2024-11-08T02:31:32,967 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf0ba59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:31:32,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees 2024-11-08T02:31:32,973 DEBUG [hconnection-0x16b5524-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-08T02:31:32,973 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=71, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:32,973 DEBUG [hconnection-0x320dea6a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,973 DEBUG [hconnection-0x39dcce67-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,974 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36174, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,974 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=71, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:32,974 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36178, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:32,974 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36190, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,983 DEBUG [hconnection-0x62cdad86-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,983 DEBUG [hconnection-0x539e15d1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:32,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:31:32,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:32,984 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36196, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:32,984 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36198, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:32,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:32,985 DEBUG [hconnection-0x3ef4d1bb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,985 DEBUG [hconnection-0x660327a6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,986 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36204, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,987 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36216, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,988 DEBUG [hconnection-0x1c4ddcd6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,989 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36224, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,991 DEBUG [hconnection-0x4196845e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,991 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:32,995 DEBUG [hconnection-0x5cf8acd5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:31:32,996 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36236, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:31:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033153001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033153003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033153004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033153005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033153006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/de1d659c20434864a05a77ebf67e8796 is 50, key is test_row_0/A:col10/1731033092980/Put/seqid=0 2024-11-08T02:31:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742037_1213 (size=12001) 2024-11-08T02:31:33,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/de1d659c20434864a05a77ebf67e8796 2024-11-08T02:31:33,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-08T02:31:33,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/87342d5417314921b4106a7519e997e7 is 50, key is test_row_0/B:col10/1731033092980/Put/seqid=0 2024-11-08T02:31:33,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742038_1214 (size=12001) 2024-11-08T02:31:33,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/87342d5417314921b4106a7519e997e7 2024-11-08T02:31:33,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033153106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033153107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033153108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033153108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033153108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,126 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-08T02:31:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:33,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:33,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(58): pid=72 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:33,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:33,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/ee5c72060d6f46daab1353cb2f06a91b is 50, key is test_row_0/C:col10/1731033092980/Put/seqid=0 2024-11-08T02:31:33,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742039_1215 (size=12001) 2024-11-08T02:31:33,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/ee5c72060d6f46daab1353cb2f06a91b 2024-11-08T02:31:33,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/de1d659c20434864a05a77ebf67e8796 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/de1d659c20434864a05a77ebf67e8796 2024-11-08T02:31:33,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/de1d659c20434864a05a77ebf67e8796, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T02:31:33,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/87342d5417314921b4106a7519e997e7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/87342d5417314921b4106a7519e997e7 2024-11-08T02:31:33,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/87342d5417314921b4106a7519e997e7, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T02:31:33,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/ee5c72060d6f46daab1353cb2f06a91b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ee5c72060d6f46daab1353cb2f06a91b 2024-11-08T02:31:33,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ee5c72060d6f46daab1353cb2f06a91b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T02:31:33,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0bc3c725c17ccfcde445d058bf2f374c in 237ms, sequenceid=12, compaction requested=false 2024-11-08T02:31:33,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-08T02:31:33,280 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-08T02:31:33,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:33,281 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:31:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:33,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:33,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/3e345698c4d74231ade6b69a981a9c07 is 50, key is test_row_0/A:col10/1731033093004/Put/seqid=0 2024-11-08T02:31:33,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:33,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033153317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033153318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033153319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033153320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033153321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742040_1216 (size=12001) 2024-11-08T02:31:33,325 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/3e345698c4d74231ade6b69a981a9c07 2024-11-08T02:31:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/185f4ea4e1a84752b7f3df568f5cf852 is 50, key is test_row_0/B:col10/1731033093004/Put/seqid=0 2024-11-08T02:31:33,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742041_1217 (size=12001) 2024-11-08T02:31:33,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033153421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033153421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033153422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033153427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033153427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-08T02:31:33,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033153622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033153624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033153624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033153629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033153630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,758 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/185f4ea4e1a84752b7f3df568f5cf852 2024-11-08T02:31:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/55a6ea6c5c2a4ed6b975c8f7f39008b1 is 50, key is test_row_0/C:col10/1731033093004/Put/seqid=0 2024-11-08T02:31:33,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742042_1218 (size=12001) 2024-11-08T02:31:33,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033153925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033153926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033153926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033153931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:33,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033153933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-08T02:31:34,169 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/55a6ea6c5c2a4ed6b975c8f7f39008b1 2024-11-08T02:31:34,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/3e345698c4d74231ade6b69a981a9c07 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3e345698c4d74231ade6b69a981a9c07 2024-11-08T02:31:34,177 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3e345698c4d74231ade6b69a981a9c07, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T02:31:34,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/185f4ea4e1a84752b7f3df568f5cf852 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/185f4ea4e1a84752b7f3df568f5cf852 2024-11-08T02:31:34,182 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/185f4ea4e1a84752b7f3df568f5cf852, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T02:31:34,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/55a6ea6c5c2a4ed6b975c8f7f39008b1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/55a6ea6c5c2a4ed6b975c8f7f39008b1 2024-11-08T02:31:34,187 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/55a6ea6c5c2a4ed6b975c8f7f39008b1, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T02:31:34,188 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 0bc3c725c17ccfcde445d058bf2f374c in 907ms, sequenceid=37, compaction requested=false 2024-11-08T02:31:34,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:34,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:34,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-11-08T02:31:34,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=72 2024-11-08T02:31:34,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71 2024-11-08T02:31:34,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2150 sec 2024-11-08T02:31:34,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees in 1.2200 sec 2024-11-08T02:31:34,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:34,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:31:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:34,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033154447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033154447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033154448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033154451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033154451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/07ddc29c9360472daeddcae5281889e1 is 50, key is test_row_0/A:col10/1731033093319/Put/seqid=0 2024-11-08T02:31:34,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742043_1219 (size=12001) 2024-11-08T02:31:34,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033154552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033154552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033154552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033154555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033154555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033154755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033154755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033154755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033154758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033154758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:34,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/07ddc29c9360472daeddcae5281889e1 2024-11-08T02:31:34,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/bbea71a5631c4371b798cafd54ee090c is 50, key is test_row_0/B:col10/1731033093319/Put/seqid=0 2024-11-08T02:31:34,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742044_1220 (size=12001) 2024-11-08T02:31:35,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033155059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033155060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033155060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033155062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033155062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-08T02:31:35,077 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 71 completed 2024-11-08T02:31:35,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees 2024-11-08T02:31:35,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-08T02:31:35,079 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=73, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:35,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-08T02:31:35,230 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2024-11-08T02:31:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:35,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] handler.RSProcedureHandler(58): pid=74 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:35,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=74 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:35,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=74 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:35,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/bbea71a5631c4371b798cafd54ee090c 2024-11-08T02:31:35,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/854cf9f83123457f8a10e3845cf90065 is 50, key is test_row_0/C:col10/1731033093319/Put/seqid=0 2024-11-08T02:31:35,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742045_1221 (size=12001) 2024-11-08T02:31:35,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/854cf9f83123457f8a10e3845cf90065 2024-11-08T02:31:35,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/07ddc29c9360472daeddcae5281889e1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/07ddc29c9360472daeddcae5281889e1 2024-11-08T02:31:35,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/07ddc29c9360472daeddcae5281889e1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T02:31:35,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/bbea71a5631c4371b798cafd54ee090c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bbea71a5631c4371b798cafd54ee090c 2024-11-08T02:31:35,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bbea71a5631c4371b798cafd54ee090c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T02:31:35,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/854cf9f83123457f8a10e3845cf90065 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/854cf9f83123457f8a10e3845cf90065 2024-11-08T02:31:35,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/854cf9f83123457f8a10e3845cf90065, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T02:31:35,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0bc3c725c17ccfcde445d058bf2f374c in 882ms, sequenceid=50, compaction requested=true 2024-11-08T02:31:35,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:35,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:35,314 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:35,314 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:35,314 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:35,314 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:35,315 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/B is initiating minor compaction (all files) 2024-11-08T02:31:35,315 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/A is initiating minor compaction (all files) 2024-11-08T02:31:35,315 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/A in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:35,315 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/B in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:35,315 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/87342d5417314921b4106a7519e997e7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/185f4ea4e1a84752b7f3df568f5cf852, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bbea71a5631c4371b798cafd54ee090c] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.2 K 2024-11-08T02:31:35,315 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/de1d659c20434864a05a77ebf67e8796, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3e345698c4d74231ade6b69a981a9c07, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/07ddc29c9360472daeddcae5281889e1] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.2 K 2024-11-08T02:31:35,315 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 87342d5417314921b4106a7519e997e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731033092980 2024-11-08T02:31:35,315 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting de1d659c20434864a05a77ebf67e8796, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731033092980 2024-11-08T02:31:35,315 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3e345698c4d74231ade6b69a981a9c07, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731033092999 2024-11-08T02:31:35,315 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 185f4ea4e1a84752b7f3df568f5cf852, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731033092999 2024-11-08T02:31:35,317 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 07ddc29c9360472daeddcae5281889e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731033093318 2024-11-08T02:31:35,317 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting bbea71a5631c4371b798cafd54ee090c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731033093318 2024-11-08T02:31:35,323 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#B#compaction#180 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:35,324 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#A#compaction#181 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:35,324 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/3ceb36dd06ec474fbef2ad1a31ab8ebf is 50, key is test_row_0/B:col10/1731033093319/Put/seqid=0 2024-11-08T02:31:35,324 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/ef7fd67a83e04f00868ec4af88f79b36 is 50, key is test_row_0/A:col10/1731033093319/Put/seqid=0 2024-11-08T02:31:35,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742047_1223 (size=12104) 2024-11-08T02:31:35,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742046_1222 (size=12104) 2024-11-08T02:31:35,334 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/3ceb36dd06ec474fbef2ad1a31ab8ebf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3ceb36dd06ec474fbef2ad1a31ab8ebf 2024-11-08T02:31:35,338 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/B of 0bc3c725c17ccfcde445d058bf2f374c into 3ceb36dd06ec474fbef2ad1a31ab8ebf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:35,338 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:35,338 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/B, priority=13, startTime=1731033095314; duration=0sec 2024-11-08T02:31:35,338 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:35,338 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:B 2024-11-08T02:31:35,338 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:35,339 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:35,339 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/C is initiating minor compaction (all files) 2024-11-08T02:31:35,339 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/C in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:35,339 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ee5c72060d6f46daab1353cb2f06a91b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/55a6ea6c5c2a4ed6b975c8f7f39008b1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/854cf9f83123457f8a10e3845cf90065] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.2 K 2024-11-08T02:31:35,340 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting ee5c72060d6f46daab1353cb2f06a91b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731033092980 2024-11-08T02:31:35,340 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 55a6ea6c5c2a4ed6b975c8f7f39008b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731033092999 2024-11-08T02:31:35,340 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 854cf9f83123457f8a10e3845cf90065, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731033093318 2024-11-08T02:31:35,347 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#C#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:35,347 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/71567d33f9ce4434afa4d1648e45f5d0 is 50, key is test_row_0/C:col10/1731033093319/Put/seqid=0 2024-11-08T02:31:35,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742048_1224 (size=12104) 2024-11-08T02:31:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-08T02:31:35,383 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2024-11-08T02:31:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:35,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:31:35,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:35,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:35,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:35,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:35,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:35,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:35,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/868c2b38ff09417d9fa6ec62c5f1642b is 50, key is test_row_0/A:col10/1731033094450/Put/seqid=0 2024-11-08T02:31:35,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742049_1225 (size=12001) 2024-11-08T02:31:35,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033155569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033155570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033155572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033155573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033155573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033155674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033155676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033155676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033155676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-08T02:31:35,733 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/ef7fd67a83e04f00868ec4af88f79b36 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/ef7fd67a83e04f00868ec4af88f79b36 2024-11-08T02:31:35,738 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/A of 0bc3c725c17ccfcde445d058bf2f374c into ef7fd67a83e04f00868ec4af88f79b36(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:35,738 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:35,738 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/A, priority=13, startTime=1731033095313; duration=0sec 2024-11-08T02:31:35,738 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:35,738 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:A 2024-11-08T02:31:35,756 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/71567d33f9ce4434afa4d1648e45f5d0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/71567d33f9ce4434afa4d1648e45f5d0 2024-11-08T02:31:35,760 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/C of 0bc3c725c17ccfcde445d058bf2f374c into 71567d33f9ce4434afa4d1648e45f5d0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:35,761 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:35,761 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/C, priority=13, startTime=1731033095314; duration=0sec 2024-11-08T02:31:35,761 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:35,761 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:C 2024-11-08T02:31:35,794 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/868c2b38ff09417d9fa6ec62c5f1642b 2024-11-08T02:31:35,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/689808a987d84906a51121115697d697 is 50, key is test_row_0/B:col10/1731033094450/Put/seqid=0 2024-11-08T02:31:35,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742050_1226 (size=12001) 2024-11-08T02:31:35,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033155877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033155877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033155878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033155879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:35,937 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:31:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-08T02:31:36,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033156182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033156182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033156183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033156183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,213 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/689808a987d84906a51121115697d697 2024-11-08T02:31:36,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/444111b4b06e477eab3e329ffb0161ac is 50, key is test_row_0/C:col10/1731033094450/Put/seqid=0 2024-11-08T02:31:36,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742051_1227 (size=12001) 2024-11-08T02:31:36,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033156581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/444111b4b06e477eab3e329ffb0161ac 2024-11-08T02:31:36,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/868c2b38ff09417d9fa6ec62c5f1642b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/868c2b38ff09417d9fa6ec62c5f1642b 2024-11-08T02:31:36,636 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/868c2b38ff09417d9fa6ec62c5f1642b, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T02:31:36,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/689808a987d84906a51121115697d697 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/689808a987d84906a51121115697d697 2024-11-08T02:31:36,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/689808a987d84906a51121115697d697, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T02:31:36,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/444111b4b06e477eab3e329ffb0161ac as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/444111b4b06e477eab3e329ffb0161ac 2024-11-08T02:31:36,645 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/444111b4b06e477eab3e329ffb0161ac, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T02:31:36,646 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0bc3c725c17ccfcde445d058bf2f374c in 1263ms, sequenceid=75, compaction requested=false 2024-11-08T02:31:36,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:36,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:36,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-11-08T02:31:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=74 2024-11-08T02:31:36,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-08T02:31:36,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-08T02:31:36,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees in 1.5710 sec 2024-11-08T02:31:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:36,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:31:36,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:36,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:36,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:36,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:36,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:36,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:36,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/1200a2d4372840adb10e9a49fbcfe65b is 50, key is test_row_0/A:col10/1731033096686/Put/seqid=0 2024-11-08T02:31:36,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033156700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033156703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033156703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033156703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742052_1228 (size=14341) 2024-11-08T02:31:36,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033156804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033156806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033156806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:36,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033156818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033157007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033157021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033157021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033157023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/1200a2d4372840adb10e9a49fbcfe65b 2024-11-08T02:31:37,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/33313dfa0f064d869bae5c23ad727457 is 50, key is test_row_0/B:col10/1731033096686/Put/seqid=0 2024-11-08T02:31:37,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742053_1229 (size=12001) 2024-11-08T02:31:37,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-08T02:31:37,183 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 73 completed 2024-11-08T02:31:37,184 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees 2024-11-08T02:31:37,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T02:31:37,185 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:37,186 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=75, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:37,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T02:31:37,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033157311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033157324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033157325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033157326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,337 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-08T02:31:37,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:37,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T02:31:37,489 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-08T02:31:37,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:37,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/33313dfa0f064d869bae5c23ad727457 2024-11-08T02:31:37,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d070676c4c5e442cb54e36238de9d12d is 50, key is test_row_0/C:col10/1731033096686/Put/seqid=0 2024-11-08T02:31:37,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742054_1230 (size=12001) 2024-11-08T02:31:37,642 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-08T02:31:37,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T02:31:37,794 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-08T02:31:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033157818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033157828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033157830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033157832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d070676c4c5e442cb54e36238de9d12d 2024-11-08T02:31:37,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/1200a2d4372840adb10e9a49fbcfe65b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/1200a2d4372840adb10e9a49fbcfe65b 2024-11-08T02:31:37,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/1200a2d4372840adb10e9a49fbcfe65b, entries=200, sequenceid=92, filesize=14.0 K 2024-11-08T02:31:37,947 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:37,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/33313dfa0f064d869bae5c23ad727457 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/33313dfa0f064d869bae5c23ad727457 2024-11-08T02:31:37,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-08T02:31:37,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:37,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:37,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/33313dfa0f064d869bae5c23ad727457, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T02:31:37,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d070676c4c5e442cb54e36238de9d12d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d070676c4c5e442cb54e36238de9d12d 2024-11-08T02:31:37,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d070676c4c5e442cb54e36238de9d12d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T02:31:37,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 0bc3c725c17ccfcde445d058bf2f374c in 1269ms, sequenceid=92, compaction requested=true 2024-11-08T02:31:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:37,956 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:37,956 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:37,957 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:37,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:37,958 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:37,958 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:37,958 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/B is initiating minor compaction (all files) 2024-11-08T02:31:37,958 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/A is initiating minor compaction (all files) 2024-11-08T02:31:37,958 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/A in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,958 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/B in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:37,958 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3ceb36dd06ec474fbef2ad1a31ab8ebf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/689808a987d84906a51121115697d697, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/33313dfa0f064d869bae5c23ad727457] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.3 K 2024-11-08T02:31:37,958 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/ef7fd67a83e04f00868ec4af88f79b36, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/868c2b38ff09417d9fa6ec62c5f1642b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/1200a2d4372840adb10e9a49fbcfe65b] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=37.5 K 2024-11-08T02:31:37,958 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ef7fd67a83e04f00868ec4af88f79b36, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731033093318 2024-11-08T02:31:37,958 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 3ceb36dd06ec474fbef2ad1a31ab8ebf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731033093318 2024-11-08T02:31:37,959 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 868c2b38ff09417d9fa6ec62c5f1642b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731033094447 2024-11-08T02:31:37,959 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 689808a987d84906a51121115697d697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731033094447 2024-11-08T02:31:37,959 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1200a2d4372840adb10e9a49fbcfe65b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731033095571 2024-11-08T02:31:37,959 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 33313dfa0f064d869bae5c23ad727457, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731033095572 2024-11-08T02:31:37,966 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#A#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:37,967 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#B#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:37,967 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/bc3975a95d2b4bfe8d1712ba2f952340 is 50, key is test_row_0/A:col10/1731033096686/Put/seqid=0 2024-11-08T02:31:37,967 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2ef298a8a3b04b70afd68185d90dd8dc is 50, key is test_row_0/B:col10/1731033096686/Put/seqid=0 2024-11-08T02:31:37,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742055_1231 (size=12207) 2024-11-08T02:31:37,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742056_1232 (size=12207) 2024-11-08T02:31:38,099 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-08T02:31:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:38,100 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:31:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:38,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:38,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:38,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/fc346501fc27487a96ec3366564f90bb is 50, key is test_row_0/A:col10/1731033096702/Put/seqid=0 2024-11-08T02:31:38,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742057_1233 (size=12001) 2024-11-08T02:31:38,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T02:31:38,385 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2ef298a8a3b04b70afd68185d90dd8dc as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2ef298a8a3b04b70afd68185d90dd8dc 2024-11-08T02:31:38,385 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/bc3975a95d2b4bfe8d1712ba2f952340 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc3975a95d2b4bfe8d1712ba2f952340 2024-11-08T02:31:38,390 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/B of 0bc3c725c17ccfcde445d058bf2f374c into 2ef298a8a3b04b70afd68185d90dd8dc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:38,390 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/A of 0bc3c725c17ccfcde445d058bf2f374c into bc3975a95d2b4bfe8d1712ba2f952340(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:38,390 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:38,390 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:38,390 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/B, priority=13, startTime=1731033097956; duration=0sec 2024-11-08T02:31:38,390 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/A, priority=13, startTime=1731033097956; duration=0sec 2024-11-08T02:31:38,390 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:38,390 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:38,390 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:A 2024-11-08T02:31:38,390 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:B 2024-11-08T02:31:38,390 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:38,391 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:38,391 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/C is initiating minor compaction (all files) 2024-11-08T02:31:38,392 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/C in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:38,392 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/71567d33f9ce4434afa4d1648e45f5d0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/444111b4b06e477eab3e329ffb0161ac, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d070676c4c5e442cb54e36238de9d12d] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.3 K 2024-11-08T02:31:38,392 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 71567d33f9ce4434afa4d1648e45f5d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731033093318 2024-11-08T02:31:38,392 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 444111b4b06e477eab3e329ffb0161ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731033094447 2024-11-08T02:31:38,393 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d070676c4c5e442cb54e36238de9d12d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731033095572 2024-11-08T02:31:38,400 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:38,400 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/e45c8e0d140d487588504a5641682f7b is 50, key is test_row_0/C:col10/1731033096686/Put/seqid=0 2024-11-08T02:31:38,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742058_1234 (size=12207) 2024-11-08T02:31:38,508 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/fc346501fc27487a96ec3366564f90bb 2024-11-08T02:31:38,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/b75bd3b713d547eeb29a1847315959a8 is 50, key is test_row_0/B:col10/1731033096702/Put/seqid=0 2024-11-08T02:31:38,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742059_1235 (size=12001) 2024-11-08T02:31:38,521 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/b75bd3b713d547eeb29a1847315959a8 2024-11-08T02:31:38,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d0a22201d33a466eacf018e6b73b61d4 is 50, key is test_row_0/C:col10/1731033096702/Put/seqid=0 2024-11-08T02:31:38,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742060_1236 (size=12001) 2024-11-08T02:31:38,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:38,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:38,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033158616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:38,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033158720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,809 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/e45c8e0d140d487588504a5641682f7b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/e45c8e0d140d487588504a5641682f7b 2024-11-08T02:31:38,814 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/C of 0bc3c725c17ccfcde445d058bf2f374c into e45c8e0d140d487588504a5641682f7b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:38,814 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:38,814 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/C, priority=13, startTime=1731033097957; duration=0sec 2024-11-08T02:31:38,814 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:38,814 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:C 2024-11-08T02:31:38,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033158820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033158835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:38,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033158835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:38,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033158838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033158923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:38,932 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d0a22201d33a466eacf018e6b73b61d4 2024-11-08T02:31:38,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/fc346501fc27487a96ec3366564f90bb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/fc346501fc27487a96ec3366564f90bb 2024-11-08T02:31:38,940 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/fc346501fc27487a96ec3366564f90bb, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T02:31:38,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/b75bd3b713d547eeb29a1847315959a8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/b75bd3b713d547eeb29a1847315959a8 2024-11-08T02:31:38,944 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/b75bd3b713d547eeb29a1847315959a8, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T02:31:38,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d0a22201d33a466eacf018e6b73b61d4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d0a22201d33a466eacf018e6b73b61d4 2024-11-08T02:31:38,948 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d0a22201d33a466eacf018e6b73b61d4, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T02:31:38,949 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0bc3c725c17ccfcde445d058bf2f374c in 849ms, sequenceid=114, compaction requested=false 2024-11-08T02:31:38,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:38,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:38,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-11-08T02:31:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-11-08T02:31:38,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-08T02:31:38,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7640 sec 2024-11-08T02:31:38,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees in 1.7670 sec 2024-11-08T02:31:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:39,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:31:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:39,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:39,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/b7b597f98a5c4b39814e0fe4f0c29a62 is 50, key is test_row_0/A:col10/1731033098604/Put/seqid=0 2024-11-08T02:31:39,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742061_1237 (size=12101) 2024-11-08T02:31:39,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033159279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-08T02:31:39,289 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 75 completed 2024-11-08T02:31:39,289 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees 2024-11-08T02:31:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-08T02:31:39,291 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:39,291 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=77, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:39,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:39,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033159382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-08T02:31:39,442 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:39,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033159585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-08T02:31:39,595 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:39,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:39,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/b7b597f98a5c4b39814e0fe4f0c29a62 2024-11-08T02:31:39,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/3575a6b646d54a258d30deb8afbc4d4a is 50, key is test_row_0/B:col10/1731033098604/Put/seqid=0 2024-11-08T02:31:39,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742062_1238 (size=12101) 2024-11-08T02:31:39,747 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:39,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:39,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033159889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-08T02:31:39,899 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:39,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:39,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:39,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:39,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,051 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:40,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/3575a6b646d54a258d30deb8afbc4d4a 2024-11-08T02:31:40,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/f0b868a9025d478989925c8e1675c9a6 is 50, key is test_row_0/C:col10/1731033098604/Put/seqid=0 2024-11-08T02:31:40,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742063_1239 (size=12101) 2024-11-08T02:31:40,203 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,356 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-08T02:31:40,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033160393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/f0b868a9025d478989925c8e1675c9a6 2024-11-08T02:31:40,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/b7b597f98a5c4b39814e0fe4f0c29a62 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/b7b597f98a5c4b39814e0fe4f0c29a62 2024-11-08T02:31:40,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/b7b597f98a5c4b39814e0fe4f0c29a62, entries=150, sequenceid=132, filesize=11.8 K 2024-11-08T02:31:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/3575a6b646d54a258d30deb8afbc4d4a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3575a6b646d54a258d30deb8afbc4d4a 2024-11-08T02:31:40,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3575a6b646d54a258d30deb8afbc4d4a, entries=150, sequenceid=132, filesize=11.8 K 2024-11-08T02:31:40,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/f0b868a9025d478989925c8e1675c9a6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f0b868a9025d478989925c8e1675c9a6 2024-11-08T02:31:40,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f0b868a9025d478989925c8e1675c9a6, entries=150, sequenceid=132, filesize=11.8 K 2024-11-08T02:31:40,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0bc3c725c17ccfcde445d058bf2f374c in 1276ms, sequenceid=132, compaction requested=true 2024-11-08T02:31:40,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:40,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:40,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:40,507 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:40,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:40,508 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:40,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:40,508 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,509 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:40,509 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/B is initiating minor compaction (all files) 2024-11-08T02:31:40,509 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/B in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,509 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2ef298a8a3b04b70afd68185d90dd8dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/b75bd3b713d547eeb29a1847315959a8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3575a6b646d54a258d30deb8afbc4d4a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.5 K 2024-11-08T02:31:40,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-08T02:31:40,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,509 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:31:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:40,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:40,510 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:40,510 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 2ef298a8a3b04b70afd68185d90dd8dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731033095572 2024-11-08T02:31:40,510 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/A is initiating minor compaction (all files) 2024-11-08T02:31:40,510 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/A in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,510 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc3975a95d2b4bfe8d1712ba2f952340, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/fc346501fc27487a96ec3366564f90bb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/b7b597f98a5c4b39814e0fe4f0c29a62] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.5 K 2024-11-08T02:31:40,511 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b75bd3b713d547eeb29a1847315959a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731033096698 2024-11-08T02:31:40,511 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting bc3975a95d2b4bfe8d1712ba2f952340, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731033095572 2024-11-08T02:31:40,511 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 3575a6b646d54a258d30deb8afbc4d4a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731033098604 2024-11-08T02:31:40,511 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting fc346501fc27487a96ec3366564f90bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731033096698 2024-11-08T02:31:40,512 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b7b597f98a5c4b39814e0fe4f0c29a62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731033098604 2024-11-08T02:31:40,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/190fe7a4fe3842cd9e5027c614995e99 is 50, key is test_row_0/A:col10/1731033099278/Put/seqid=0 2024-11-08T02:31:40,537 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#B#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:40,537 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#A#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:40,538 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/1e61576645014707b062967c220138ad is 50, key is test_row_0/B:col10/1731033098604/Put/seqid=0 2024-11-08T02:31:40,538 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/941faf9f04c8458ba0494b9c38a40021 is 50, key is test_row_0/A:col10/1731033098604/Put/seqid=0 2024-11-08T02:31:40,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742064_1240 (size=12151) 2024-11-08T02:31:40,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742065_1241 (size=12409) 2024-11-08T02:31:40,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742066_1242 (size=12409) 2024-11-08T02:31:40,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:40,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033160858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033160858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033160859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033160859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,939 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/190fe7a4fe3842cd9e5027c614995e99 2024-11-08T02:31:40,946 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/1e61576645014707b062967c220138ad as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1e61576645014707b062967c220138ad 2024-11-08T02:31:40,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/723f7debeef449c0b2692fd2ef10ab1a is 50, key is test_row_0/B:col10/1731033099278/Put/seqid=0 2024-11-08T02:31:40,951 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/941faf9f04c8458ba0494b9c38a40021 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/941faf9f04c8458ba0494b9c38a40021 2024-11-08T02:31:40,953 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/B of 0bc3c725c17ccfcde445d058bf2f374c into 1e61576645014707b062967c220138ad(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:40,953 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:40,953 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/B, priority=13, startTime=1731033100507; duration=0sec 2024-11-08T02:31:40,953 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:40,953 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:B 2024-11-08T02:31:40,953 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:40,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742067_1243 (size=12151) 2024-11-08T02:31:40,956 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:40,956 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/C is initiating minor compaction (all files) 2024-11-08T02:31:40,956 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/C in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:40,956 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/e45c8e0d140d487588504a5641682f7b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d0a22201d33a466eacf018e6b73b61d4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f0b868a9025d478989925c8e1675c9a6] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.5 K 2024-11-08T02:31:40,957 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e45c8e0d140d487588504a5641682f7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731033095572 2024-11-08T02:31:40,959 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/A of 0bc3c725c17ccfcde445d058bf2f374c into 941faf9f04c8458ba0494b9c38a40021(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:40,959 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:40,959 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/A, priority=13, startTime=1731033100507; duration=0sec 2024-11-08T02:31:40,959 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:40,959 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:A 2024-11-08T02:31:40,959 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting d0a22201d33a466eacf018e6b73b61d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731033096698 2024-11-08T02:31:40,960 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f0b868a9025d478989925c8e1675c9a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731033098604 2024-11-08T02:31:40,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033160960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033160961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033160963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033160964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:40,969 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:40,969 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/8791ed7ed83e42b6847499aa86672e43 is 50, key is test_row_0/C:col10/1731033098604/Put/seqid=0 2024-11-08T02:31:40,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742068_1244 (size=12409) 2024-11-08T02:31:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033161164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033161164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033161166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033161167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,356 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/723f7debeef449c0b2692fd2ef10ab1a 2024-11-08T02:31:41,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/dac81e6bd0d04f1ba44c371e7fef4964 is 50, key is test_row_0/C:col10/1731033099278/Put/seqid=0 2024-11-08T02:31:41,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742069_1245 (size=12151) 2024-11-08T02:31:41,386 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/8791ed7ed83e42b6847499aa86672e43 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/8791ed7ed83e42b6847499aa86672e43 2024-11-08T02:31:41,390 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/C of 0bc3c725c17ccfcde445d058bf2f374c into 8791ed7ed83e42b6847499aa86672e43(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:41,390 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:41,390 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/C, priority=13, startTime=1731033100508; duration=0sec 2024-11-08T02:31:41,390 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:41,390 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:C 2024-11-08T02:31:41,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-08T02:31:41,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033161396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033161468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033161468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033161469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033161469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,767 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/dac81e6bd0d04f1ba44c371e7fef4964 2024-11-08T02:31:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/190fe7a4fe3842cd9e5027c614995e99 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/190fe7a4fe3842cd9e5027c614995e99 2024-11-08T02:31:41,774 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/190fe7a4fe3842cd9e5027c614995e99, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T02:31:41,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/723f7debeef449c0b2692fd2ef10ab1a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/723f7debeef449c0b2692fd2ef10ab1a 2024-11-08T02:31:41,779 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/723f7debeef449c0b2692fd2ef10ab1a, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T02:31:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/dac81e6bd0d04f1ba44c371e7fef4964 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/dac81e6bd0d04f1ba44c371e7fef4964 2024-11-08T02:31:41,785 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/dac81e6bd0d04f1ba44c371e7fef4964, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T02:31:41,787 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0bc3c725c17ccfcde445d058bf2f374c in 1278ms, sequenceid=153, compaction requested=false 2024-11-08T02:31:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:41,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-11-08T02:31:41,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-11-08T02:31:41,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-11-08T02:31:41,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4970 sec 2024-11-08T02:31:41,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees in 2.5000 sec 2024-11-08T02:31:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:41,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:31:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:41,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:41,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:41,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:41,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:41,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:41,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/9f157a071ee84332864816e81c1106b4 is 50, key is test_row_0/A:col10/1731033101970/Put/seqid=0 2024-11-08T02:31:41,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742070_1246 (size=14541) 2024-11-08T02:31:41,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033161985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033161985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033161986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:41,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033161987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033162088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033162089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033162089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033162089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033162290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033162291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033162292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033162292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/9f157a071ee84332864816e81c1106b4 2024-11-08T02:31:42,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/064511b06ba145c597c70d9e23005b51 is 50, key is test_row_0/B:col10/1731033101970/Put/seqid=0 2024-11-08T02:31:42,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742071_1247 (size=12151) 2024-11-08T02:31:42,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033162593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033162594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033162595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033162595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:42,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/064511b06ba145c597c70d9e23005b51 2024-11-08T02:31:42,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/ec6b76a8af4845afb22e0888f8b40ef3 is 50, key is test_row_0/C:col10/1731033101970/Put/seqid=0 2024-11-08T02:31:42,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742072_1248 (size=12151) 2024-11-08T02:31:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033163098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033163099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033163100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:43,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033163100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/ec6b76a8af4845afb22e0888f8b40ef3 2024-11-08T02:31:43,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/9f157a071ee84332864816e81c1106b4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/9f157a071ee84332864816e81c1106b4 2024-11-08T02:31:43,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/9f157a071ee84332864816e81c1106b4, entries=200, sequenceid=173, filesize=14.2 K 2024-11-08T02:31:43,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/064511b06ba145c597c70d9e23005b51 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/064511b06ba145c597c70d9e23005b51 2024-11-08T02:31:43,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/064511b06ba145c597c70d9e23005b51, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T02:31:43,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/ec6b76a8af4845afb22e0888f8b40ef3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ec6b76a8af4845afb22e0888f8b40ef3 2024-11-08T02:31:43,221 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ec6b76a8af4845afb22e0888f8b40ef3, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T02:31:43,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 0bc3c725c17ccfcde445d058bf2f374c in 1251ms, sequenceid=173, compaction requested=true 2024-11-08T02:31:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:43,222 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:43,222 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:43,223 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:43,223 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/A is initiating minor compaction (all files) 2024-11-08T02:31:43,223 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/A in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,223 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/941faf9f04c8458ba0494b9c38a40021, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/190fe7a4fe3842cd9e5027c614995e99, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/9f157a071ee84332864816e81c1106b4] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=38.2 K 2024-11-08T02:31:43,223 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:43,224 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/B is initiating minor compaction (all files) 2024-11-08T02:31:43,224 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/B in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,224 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1e61576645014707b062967c220138ad, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/723f7debeef449c0b2692fd2ef10ab1a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/064511b06ba145c597c70d9e23005b51] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.9 K 2024-11-08T02:31:43,224 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 941faf9f04c8458ba0494b9c38a40021, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731033098604 2024-11-08T02:31:43,224 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1e61576645014707b062967c220138ad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731033098604 2024-11-08T02:31:43,224 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 190fe7a4fe3842cd9e5027c614995e99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731033099267 2024-11-08T02:31:43,224 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 723f7debeef449c0b2692fd2ef10ab1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731033099267 2024-11-08T02:31:43,224 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9f157a071ee84332864816e81c1106b4, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731033100858 2024-11-08T02:31:43,225 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 064511b06ba145c597c70d9e23005b51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731033100858 2024-11-08T02:31:43,232 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:43,232 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/33a3be06012c4ddd9c88a7c5af79a978 is 50, key is test_row_0/A:col10/1731033101970/Put/seqid=0 2024-11-08T02:31:43,235 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#B#compaction#208 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:43,236 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/1d02000ed8584647b16e1a1c312cfc6c is 50, key is test_row_0/B:col10/1731033101970/Put/seqid=0 2024-11-08T02:31:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742073_1249 (size=12561) 2024-11-08T02:31:43,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742074_1250 (size=12561) 2024-11-08T02:31:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-08T02:31:43,395 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 77 completed 2024-11-08T02:31:43,396 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees 2024-11-08T02:31:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-08T02:31:43,397 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:43,397 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=79, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:43,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:43,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:43,407 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:31:43,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:43,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:43,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:43,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:43,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:43,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:43,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/bea90774e97a49eca3959cb4c6c3b11f is 50, key is test_row_0/A:col10/1731033103406/Put/seqid=0 2024-11-08T02:31:43,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742075_1251 (size=12151) 2024-11-08T02:31:43,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:43,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033163452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-08T02:31:43,549 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:43,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033163555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,645 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/33a3be06012c4ddd9c88a7c5af79a978 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/33a3be06012c4ddd9c88a7c5af79a978 2024-11-08T02:31:43,647 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/1d02000ed8584647b16e1a1c312cfc6c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1d02000ed8584647b16e1a1c312cfc6c 2024-11-08T02:31:43,649 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/A of 0bc3c725c17ccfcde445d058bf2f374c into 33a3be06012c4ddd9c88a7c5af79a978(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:43,650 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:43,650 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/A, priority=13, startTime=1731033103222; duration=0sec 2024-11-08T02:31:43,650 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:43,650 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:A 2024-11-08T02:31:43,650 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:43,651 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:43,651 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/C is initiating minor compaction (all files) 2024-11-08T02:31:43,651 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/C in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,651 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/8791ed7ed83e42b6847499aa86672e43, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/dac81e6bd0d04f1ba44c371e7fef4964, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ec6b76a8af4845afb22e0888f8b40ef3] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=35.9 K 2024-11-08T02:31:43,651 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8791ed7ed83e42b6847499aa86672e43, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731033098604 2024-11-08T02:31:43,651 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/B of 0bc3c725c17ccfcde445d058bf2f374c into 1d02000ed8584647b16e1a1c312cfc6c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:43,651 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting dac81e6bd0d04f1ba44c371e7fef4964, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731033099267 2024-11-08T02:31:43,651 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:43,651 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/B, priority=13, startTime=1731033103222; duration=0sec 2024-11-08T02:31:43,651 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:43,651 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:B 2024-11-08T02:31:43,652 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ec6b76a8af4845afb22e0888f8b40ef3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731033100858 2024-11-08T02:31:43,658 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#C#compaction#210 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:43,658 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/a43cd558a0214443bdcb15bb367b122a is 50, key is test_row_0/C:col10/1731033101970/Put/seqid=0 2024-11-08T02:31:43,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742076_1252 (size=12561) 2024-11-08T02:31:43,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-08T02:31:43,702 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:43,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033163758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/bea90774e97a49eca3959cb4c6c3b11f 2024-11-08T02:31:43,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2d0a5226720e4b8498bdb29cb23080e2 is 50, key is test_row_0/B:col10/1731033103406/Put/seqid=0 2024-11-08T02:31:43,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742077_1253 (size=12151) 2024-11-08T02:31:43,854 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:43,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:43,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:43,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-08T02:31:44,007 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033164060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,068 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/a43cd558a0214443bdcb15bb367b122a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/a43cd558a0214443bdcb15bb367b122a 2024-11-08T02:31:44,072 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/C of 0bc3c725c17ccfcde445d058bf2f374c into a43cd558a0214443bdcb15bb367b122a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:44,072 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:44,072 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/C, priority=13, startTime=1731033103222; duration=0sec 2024-11-08T02:31:44,072 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:44,072 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:C 2024-11-08T02:31:44,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033164103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033164104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033164105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:44,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033164111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,159 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:44,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2d0a5226720e4b8498bdb29cb23080e2 2024-11-08T02:31:44,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/921343e7a90b4c45862912820e3f83a4 is 50, key is test_row_0/C:col10/1731033103406/Put/seqid=0 2024-11-08T02:31:44,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742078_1254 (size=12151) 2024-11-08T02:31:44,311 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,464 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-08T02:31:44,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033164564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,616 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:44,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/921343e7a90b4c45862912820e3f83a4 2024-11-08T02:31:44,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/bea90774e97a49eca3959cb4c6c3b11f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bea90774e97a49eca3959cb4c6c3b11f 2024-11-08T02:31:44,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bea90774e97a49eca3959cb4c6c3b11f, entries=150, sequenceid=193, filesize=11.9 K 2024-11-08T02:31:44,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2d0a5226720e4b8498bdb29cb23080e2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2d0a5226720e4b8498bdb29cb23080e2 2024-11-08T02:31:44,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2d0a5226720e4b8498bdb29cb23080e2, entries=150, sequenceid=193, filesize=11.9 K 2024-11-08T02:31:44,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/921343e7a90b4c45862912820e3f83a4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/921343e7a90b4c45862912820e3f83a4 2024-11-08T02:31:44,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/921343e7a90b4c45862912820e3f83a4, entries=150, sequenceid=193, filesize=11.9 K 2024-11-08T02:31:44,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0bc3c725c17ccfcde445d058bf2f374c in 1274ms, sequenceid=193, compaction requested=false 2024-11-08T02:31:44,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:44,769 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:44,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-08T02:31:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:44,770 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:31:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:44,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/8e4f6ba0c45b427088b414a6d47284be is 50, key is test_row_0/A:col10/1731033103446/Put/seqid=0 2024-11-08T02:31:44,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742079_1255 (size=12151) 2024-11-08T02:31:45,178 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/8e4f6ba0c45b427088b414a6d47284be 2024-11-08T02:31:45,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/5b87fcabcd8d4edead6f95c49b67be07 is 50, key is test_row_0/B:col10/1731033103446/Put/seqid=0 2024-11-08T02:31:45,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742080_1256 (size=12151) 2024-11-08T02:31:45,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-08T02:31:45,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:45,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:45,591 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/5b87fcabcd8d4edead6f95c49b67be07 2024-11-08T02:31:45,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/f9d2b368966a41ab92a9102c141551fe is 50, key is test_row_0/C:col10/1731033103446/Put/seqid=0 2024-11-08T02:31:45,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742081_1257 (size=12151) 2024-11-08T02:31:45,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033165614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:45,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033165718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:45,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:45,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033165921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,002 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/f9d2b368966a41ab92a9102c141551fe 2024-11-08T02:31:46,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/8e4f6ba0c45b427088b414a6d47284be as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8e4f6ba0c45b427088b414a6d47284be 2024-11-08T02:31:46,010 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8e4f6ba0c45b427088b414a6d47284be, entries=150, sequenceid=212, filesize=11.9 K 2024-11-08T02:31:46,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/5b87fcabcd8d4edead6f95c49b67be07 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5b87fcabcd8d4edead6f95c49b67be07 2024-11-08T02:31:46,015 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5b87fcabcd8d4edead6f95c49b67be07, entries=150, sequenceid=212, filesize=11.9 K 2024-11-08T02:31:46,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/f9d2b368966a41ab92a9102c141551fe as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f9d2b368966a41ab92a9102c141551fe 2024-11-08T02:31:46,019 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f9d2b368966a41ab92a9102c141551fe, entries=150, sequenceid=212, filesize=11.9 K 2024-11-08T02:31:46,020 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0bc3c725c17ccfcde445d058bf2f374c in 1250ms, sequenceid=212, compaction requested=true 2024-11-08T02:31:46,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:46,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:46,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-11-08T02:31:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-11-08T02:31:46,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-08T02:31:46,022 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-11-08T02:31:46,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees in 2.6260 sec 2024-11-08T02:31:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:46,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:31:46,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:46,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:46,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:46,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:46,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:46,116 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:46,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/3d647ee772404e6ca64679106030f6cf is 50, key is test_row_0/A:col10/1731033105613/Put/seqid=0 2024-11-08T02:31:46,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742082_1258 (size=12151) 2024-11-08T02:31:46,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033166129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033166130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033166131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033166131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033166225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033166232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033166234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033166235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033166235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033166436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033166437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033166438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033166438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/3d647ee772404e6ca64679106030f6cf 2024-11-08T02:31:46,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/91413fad377843c7bae9e6609e896dce is 50, key is test_row_0/B:col10/1731033105613/Put/seqid=0 2024-11-08T02:31:46,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742083_1259 (size=12151) 2024-11-08T02:31:46,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033166732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033166739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033166740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033166740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033166743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:46,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/91413fad377843c7bae9e6609e896dce 2024-11-08T02:31:46,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/c8c9ae6b894b49c39cba935bfd791505 is 50, key is test_row_0/C:col10/1731033105613/Put/seqid=0 2024-11-08T02:31:46,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742084_1260 (size=12151) 2024-11-08T02:31:46,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/c8c9ae6b894b49c39cba935bfd791505 2024-11-08T02:31:46,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/3d647ee772404e6ca64679106030f6cf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3d647ee772404e6ca64679106030f6cf 2024-11-08T02:31:46,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3d647ee772404e6ca64679106030f6cf, entries=150, sequenceid=233, filesize=11.9 K 2024-11-08T02:31:46,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/91413fad377843c7bae9e6609e896dce as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/91413fad377843c7bae9e6609e896dce 2024-11-08T02:31:46,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/91413fad377843c7bae9e6609e896dce, entries=150, sequenceid=233, filesize=11.9 K 2024-11-08T02:31:46,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/c8c9ae6b894b49c39cba935bfd791505 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c8c9ae6b894b49c39cba935bfd791505 2024-11-08T02:31:46,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c8c9ae6b894b49c39cba935bfd791505, entries=150, sequenceid=233, filesize=11.9 K 2024-11-08T02:31:46,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 0bc3c725c17ccfcde445d058bf2f374c in 855ms, sequenceid=233, compaction requested=true 2024-11-08T02:31:46,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:46,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:46,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:46,971 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:31:46,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:46,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:46,971 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:31:46,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:46,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:46,972 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:31:46,972 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/A is initiating minor compaction (all files) 2024-11-08T02:31:46,972 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:31:46,972 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/A in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:46,972 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/B is initiating minor compaction (all files) 2024-11-08T02:31:46,972 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/33a3be06012c4ddd9c88a7c5af79a978, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bea90774e97a49eca3959cb4c6c3b11f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8e4f6ba0c45b427088b414a6d47284be, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3d647ee772404e6ca64679106030f6cf] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=47.9 K 2024-11-08T02:31:46,972 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/B in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:46,972 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1d02000ed8584647b16e1a1c312cfc6c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2d0a5226720e4b8498bdb29cb23080e2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5b87fcabcd8d4edead6f95c49b67be07, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/91413fad377843c7bae9e6609e896dce] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=47.9 K 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 33a3be06012c4ddd9c88a7c5af79a978, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731033100858 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1d02000ed8584647b16e1a1c312cfc6c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731033100858 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 2d0a5226720e4b8498bdb29cb23080e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731033101985 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting bea90774e97a49eca3959cb4c6c3b11f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731033101985 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 5b87fcabcd8d4edead6f95c49b67be07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731033103436 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8e4f6ba0c45b427088b414a6d47284be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731033103436 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3d647ee772404e6ca64679106030f6cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033105610 2024-11-08T02:31:46,973 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 91413fad377843c7bae9e6609e896dce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033105610 2024-11-08T02:31:46,983 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:46,983 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#B#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:46,983 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/95611e302dcf4c579ee1acf981f55fd7 is 50, key is test_row_0/A:col10/1731033105613/Put/seqid=0 2024-11-08T02:31:46,983 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/ce044f11cc47474a899f363b0345a375 is 50, key is test_row_0/B:col10/1731033105613/Put/seqid=0 2024-11-08T02:31:46,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742085_1261 (size=12697) 2024-11-08T02:31:47,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742086_1262 (size=12697) 2024-11-08T02:31:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:47,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:31:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:47,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/08671855036a4f9aa7493b5d70bc3fa5 is 50, key is test_row_0/A:col10/1731033106129/Put/seqid=0 2024-11-08T02:31:47,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742087_1263 (size=12151) 2024-11-08T02:31:47,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033167288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033167293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033167293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033167294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033167394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033167397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033167398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033167398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,405 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/ce044f11cc47474a899f363b0345a375 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ce044f11cc47474a899f363b0345a375 2024-11-08T02:31:47,409 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/B of 0bc3c725c17ccfcde445d058bf2f374c into ce044f11cc47474a899f363b0345a375(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:47,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:47,409 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/B, priority=12, startTime=1731033106971; duration=0sec 2024-11-08T02:31:47,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:47,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:B 2024-11-08T02:31:47,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:31:47,411 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:31:47,411 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/C is initiating minor compaction (all files) 2024-11-08T02:31:47,411 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/C in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:47,411 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/a43cd558a0214443bdcb15bb367b122a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/921343e7a90b4c45862912820e3f83a4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f9d2b368966a41ab92a9102c141551fe, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c8c9ae6b894b49c39cba935bfd791505] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=47.9 K 2024-11-08T02:31:47,412 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a43cd558a0214443bdcb15bb367b122a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731033100858 2024-11-08T02:31:47,412 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 921343e7a90b4c45862912820e3f83a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731033101985 2024-11-08T02:31:47,413 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f9d2b368966a41ab92a9102c141551fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731033103436 2024-11-08T02:31:47,413 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/95611e302dcf4c579ee1acf981f55fd7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/95611e302dcf4c579ee1acf981f55fd7 2024-11-08T02:31:47,413 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c8c9ae6b894b49c39cba935bfd791505, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033105610 2024-11-08T02:31:47,418 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/A of 0bc3c725c17ccfcde445d058bf2f374c into 95611e302dcf4c579ee1acf981f55fd7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:47,418 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:47,418 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/A, priority=12, startTime=1731033106970; duration=0sec 2024-11-08T02:31:47,418 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:47,418 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:A 2024-11-08T02:31:47,424 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#C#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:47,425 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/fcfe1a968cf948bcac21806ccd6c9faa is 50, key is test_row_0/C:col10/1731033105613/Put/seqid=0 2024-11-08T02:31:47,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742088_1264 (size=12697) 2024-11-08T02:31:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-08T02:31:47,501 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 79 completed 2024-11-08T02:31:47,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2024-11-08T02:31:47,504 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T02:31:47,504 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:47,504 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:47,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033167597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033167601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033167602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T02:31:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033167602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/08671855036a4f9aa7493b5d70bc3fa5 2024-11-08T02:31:47,655 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-08T02:31:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:47,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/43bdad14cde14427bc0521b7e1e48279 is 50, key is test_row_0/B:col10/1731033106129/Put/seqid=0 2024-11-08T02:31:47,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742089_1265 (size=12151) 2024-11-08T02:31:47,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033167742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T02:31:47,807 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-08T02:31:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:47,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,853 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/fcfe1a968cf948bcac21806ccd6c9faa as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/fcfe1a968cf948bcac21806ccd6c9faa 2024-11-08T02:31:47,857 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/C of 0bc3c725c17ccfcde445d058bf2f374c into fcfe1a968cf948bcac21806ccd6c9faa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:47,857 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:47,857 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/C, priority=12, startTime=1731033106971; duration=0sec 2024-11-08T02:31:47,857 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:47,857 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:C 2024-11-08T02:31:47,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033167902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033167904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033167905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:47,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033167905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,959 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:47,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-08T02:31:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:47,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/43bdad14cde14427bc0521b7e1e48279 2024-11-08T02:31:48,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/b91fcb458f1c447291ad3a07bc0ac75d is 50, key is test_row_0/C:col10/1731033106129/Put/seqid=0 2024-11-08T02:31:48,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742090_1266 (size=12151) 2024-11-08T02:31:48,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T02:31:48,111 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-08T02:31:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:48,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,264 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-08T02:31:48,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:48,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:48,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:48,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033168404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033168409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033168410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033168411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,416 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-08T02:31:48,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:48,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:48,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:48,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/b91fcb458f1c447291ad3a07bc0ac75d 2024-11-08T02:31:48,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/08671855036a4f9aa7493b5d70bc3fa5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/08671855036a4f9aa7493b5d70bc3fa5 2024-11-08T02:31:48,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/08671855036a4f9aa7493b5d70bc3fa5, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T02:31:48,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/43bdad14cde14427bc0521b7e1e48279 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/43bdad14cde14427bc0521b7e1e48279 2024-11-08T02:31:48,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/43bdad14cde14427bc0521b7e1e48279, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T02:31:48,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/b91fcb458f1c447291ad3a07bc0ac75d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b91fcb458f1c447291ad3a07bc0ac75d 2024-11-08T02:31:48,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b91fcb458f1c447291ad3a07bc0ac75d, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T02:31:48,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 0bc3c725c17ccfcde445d058bf2f374c in 1248ms, sequenceid=249, compaction requested=false 2024-11-08T02:31:48,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:48,568 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:48,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-08T02:31:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:48,569 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:31:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:48,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/31e2cf6a830f4499ae118ec63e4c8416 is 50, key is test_row_0/A:col10/1731033107293/Put/seqid=0 2024-11-08T02:31:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742091_1267 (size=12301) 2024-11-08T02:31:48,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T02:31:48,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/31e2cf6a830f4499ae118ec63e4c8416 2024-11-08T02:31:48,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/08a153b5a7e04f9ebaf34b430489bcdf is 50, key is test_row_0/B:col10/1731033107293/Put/seqid=0 2024-11-08T02:31:49,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742092_1268 (size=12301) 2024-11-08T02:31:49,408 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/08a153b5a7e04f9ebaf34b430489bcdf 2024-11-08T02:31:49,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:49,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d7e2d2b294864a7a8f871a9a72261be7 is 50, key is test_row_0/C:col10/1731033107293/Put/seqid=0 2024-11-08T02:31:49,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742093_1269 (size=12301) 2024-11-08T02:31:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033169427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033169427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033169428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033169427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033169530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033169530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033169530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033169531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T02:31:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033169733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033169733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033169733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033169733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033169746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:49,748 DEBUG [Thread-1002 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4134 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:31:49,819 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d7e2d2b294864a7a8f871a9a72261be7 2024-11-08T02:31:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/31e2cf6a830f4499ae118ec63e4c8416 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/31e2cf6a830f4499ae118ec63e4c8416 2024-11-08T02:31:49,826 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/31e2cf6a830f4499ae118ec63e4c8416, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:31:49,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/08a153b5a7e04f9ebaf34b430489bcdf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/08a153b5a7e04f9ebaf34b430489bcdf 2024-11-08T02:31:49,829 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/08a153b5a7e04f9ebaf34b430489bcdf, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:31:49,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d7e2d2b294864a7a8f871a9a72261be7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d7e2d2b294864a7a8f871a9a72261be7 2024-11-08T02:31:49,834 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d7e2d2b294864a7a8f871a9a72261be7, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:31:49,835 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 0bc3c725c17ccfcde445d058bf2f374c in 1265ms, sequenceid=273, compaction requested=true 2024-11-08T02:31:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-11-08T02:31:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-11-08T02:31:49,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-08T02:31:49,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3320 sec 2024-11-08T02:31:49,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 2.3350 sec 2024-11-08T02:31:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:50,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:31:50,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:50,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:50,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:50,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:50,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:50,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:50,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/197583de9a724f45a1a23602c3743aba is 50, key is test_row_0/A:col10/1731033110037/Put/seqid=0 2024-11-08T02:31:50,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742094_1270 (size=14741) 2024-11-08T02:31:50,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033170050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033170052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033170052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033170053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033170154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033170155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033170155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033170156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033170357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033170358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033170359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033170359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/197583de9a724f45a1a23602c3743aba 2024-11-08T02:31:50,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/90aa40af910a412c9d672d4084c0e5eb is 50, key is test_row_0/B:col10/1731033110037/Put/seqid=0 2024-11-08T02:31:50,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742095_1271 (size=12301) 2024-11-08T02:31:50,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033170663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033170664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033170664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:50,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033170664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:50,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/90aa40af910a412c9d672d4084c0e5eb 2024-11-08T02:31:50,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/7a64afc548ff4a13b770830c603cc493 is 50, key is test_row_0/C:col10/1731033110037/Put/seqid=0 2024-11-08T02:31:50,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742096_1272 (size=12301) 2024-11-08T02:31:51,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033171168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:51,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033171169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:51,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033171170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033171171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:51,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/7a64afc548ff4a13b770830c603cc493 2024-11-08T02:31:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/197583de9a724f45a1a23602c3743aba as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/197583de9a724f45a1a23602c3743aba 2024-11-08T02:31:51,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/197583de9a724f45a1a23602c3743aba, entries=200, sequenceid=291, filesize=14.4 K 2024-11-08T02:31:51,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/90aa40af910a412c9d672d4084c0e5eb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/90aa40af910a412c9d672d4084c0e5eb 2024-11-08T02:31:51,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/90aa40af910a412c9d672d4084c0e5eb, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T02:31:51,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/7a64afc548ff4a13b770830c603cc493 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/7a64afc548ff4a13b770830c603cc493 2024-11-08T02:31:51,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/7a64afc548ff4a13b770830c603cc493, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T02:31:51,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0bc3c725c17ccfcde445d058bf2f374c in 1271ms, sequenceid=291, compaction requested=true 2024-11-08T02:31:51,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:51,310 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:31:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:51,310 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:31:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:51,311 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:31:51,311 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51890 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:31:51,311 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/A is initiating minor compaction (all files) 2024-11-08T02:31:51,311 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/B is initiating minor compaction (all files) 2024-11-08T02:31:51,311 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/A in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:51,311 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/B in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:51,311 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/95611e302dcf4c579ee1acf981f55fd7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/08671855036a4f9aa7493b5d70bc3fa5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/31e2cf6a830f4499ae118ec63e4c8416, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/197583de9a724f45a1a23602c3743aba] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=50.7 K 2024-11-08T02:31:51,311 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ce044f11cc47474a899f363b0345a375, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/43bdad14cde14427bc0521b7e1e48279, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/08a153b5a7e04f9ebaf34b430489bcdf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/90aa40af910a412c9d672d4084c0e5eb] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=48.3 K 2024-11-08T02:31:51,311 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting ce044f11cc47474a899f363b0345a375, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033105610 2024-11-08T02:31:51,312 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 95611e302dcf4c579ee1acf981f55fd7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033105610 2024-11-08T02:31:51,312 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 08671855036a4f9aa7493b5d70bc3fa5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731033106129 2024-11-08T02:31:51,312 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 43bdad14cde14427bc0521b7e1e48279, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731033106129 2024-11-08T02:31:51,312 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 31e2cf6a830f4499ae118ec63e4c8416, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731033107258 2024-11-08T02:31:51,312 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 08a153b5a7e04f9ebaf34b430489bcdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731033107258 2024-11-08T02:31:51,312 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 90aa40af910a412c9d672d4084c0e5eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033109427 2024-11-08T02:31:51,312 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 197583de9a724f45a1a23602c3743aba, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033109422 2024-11-08T02:31:51,325 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#B#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:51,325 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/281ef86a2e904dab89c6430d389d1092 is 50, key is test_row_0/B:col10/1731033110037/Put/seqid=0 2024-11-08T02:31:51,326 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:51,326 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/a65ef276265e42b0bcffdd3032178329 is 50, key is test_row_0/A:col10/1731033110037/Put/seqid=0 2024-11-08T02:31:51,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742098_1274 (size=12983) 2024-11-08T02:31:51,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742097_1273 (size=12983) 2024-11-08T02:31:51,339 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/a65ef276265e42b0bcffdd3032178329 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/a65ef276265e42b0bcffdd3032178329 2024-11-08T02:31:51,344 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/A of 0bc3c725c17ccfcde445d058bf2f374c into a65ef276265e42b0bcffdd3032178329(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:51,344 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:51,344 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/A, priority=12, startTime=1731033111309; duration=0sec 2024-11-08T02:31:51,344 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:51,344 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:A 2024-11-08T02:31:51,344 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:31:51,345 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:31:51,345 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/C is initiating minor compaction (all files) 2024-11-08T02:31:51,345 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/C in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:51,346 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/fcfe1a968cf948bcac21806ccd6c9faa, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b91fcb458f1c447291ad3a07bc0ac75d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d7e2d2b294864a7a8f871a9a72261be7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/7a64afc548ff4a13b770830c603cc493] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=48.3 K 2024-11-08T02:31:51,347 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting fcfe1a968cf948bcac21806ccd6c9faa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033105610 2024-11-08T02:31:51,347 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b91fcb458f1c447291ad3a07bc0ac75d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731033106129 2024-11-08T02:31:51,347 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d7e2d2b294864a7a8f871a9a72261be7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731033107258 2024-11-08T02:31:51,347 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7a64afc548ff4a13b770830c603cc493, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033109427 2024-11-08T02:31:51,358 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:51,358 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/b8ae9b961b33434c9880a51be36084b3 is 50, key is test_row_0/C:col10/1731033110037/Put/seqid=0 2024-11-08T02:31:51,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742099_1275 (size=12983) 2024-11-08T02:31:51,378 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/b8ae9b961b33434c9880a51be36084b3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b8ae9b961b33434c9880a51be36084b3 2024-11-08T02:31:51,383 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/C of 0bc3c725c17ccfcde445d058bf2f374c into b8ae9b961b33434c9880a51be36084b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:51,384 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:51,384 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/C, priority=12, startTime=1731033111310; duration=0sec 2024-11-08T02:31:51,384 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:51,384 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:C 2024-11-08T02:31:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-08T02:31:51,608 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 81 completed 2024-11-08T02:31:51,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-11-08T02:31:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-08T02:31:51,610 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:51,610 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:51,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-08T02:31:51,743 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/281ef86a2e904dab89c6430d389d1092 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/281ef86a2e904dab89c6430d389d1092 2024-11-08T02:31:51,747 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/B of 0bc3c725c17ccfcde445d058bf2f374c into 281ef86a2e904dab89c6430d389d1092(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:51,747 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:51,747 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/B, priority=12, startTime=1731033111310; duration=0sec 2024-11-08T02:31:51,747 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:51,747 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:B 2024-11-08T02:31:51,761 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:51,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-08T02:31:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:51,762 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:31:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:51,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:51,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:51,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:51,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:51,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/8f9fd9b647ed480d9c7caefa7fbfb720 is 50, key is test_row_0/A:col10/1731033110051/Put/seqid=0 2024-11-08T02:31:51,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742100_1276 (size=12301) 2024-11-08T02:31:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-08T02:31:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:52,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:52,186 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/8f9fd9b647ed480d9c7caefa7fbfb720 2024-11-08T02:31:52,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033172186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033172188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033172188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033172188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2824e274cb5249feae5e8c6d747d8a0d is 50, key is test_row_0/B:col10/1731033110051/Put/seqid=0 2024-11-08T02:31:52,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742101_1277 (size=12301) 2024-11-08T02:31:52,200 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2824e274cb5249feae5e8c6d747d8a0d 2024-11-08T02:31:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/4c1f5d444c314216820ba688f0cf01e6 is 50, key is test_row_0/C:col10/1731033110051/Put/seqid=0 2024-11-08T02:31:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-08T02:31:52,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742102_1278 (size=12301) 2024-11-08T02:31:52,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033172289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033172291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033172291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033172292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033172492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033172493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033172494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033172495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/4c1f5d444c314216820ba688f0cf01e6 2024-11-08T02:31:52,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/8f9fd9b647ed480d9c7caefa7fbfb720 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8f9fd9b647ed480d9c7caefa7fbfb720 2024-11-08T02:31:52,620 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8f9fd9b647ed480d9c7caefa7fbfb720, entries=150, sequenceid=312, filesize=12.0 K 2024-11-08T02:31:52,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/2824e274cb5249feae5e8c6d747d8a0d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2824e274cb5249feae5e8c6d747d8a0d 2024-11-08T02:31:52,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2824e274cb5249feae5e8c6d747d8a0d, entries=150, sequenceid=312, filesize=12.0 K 2024-11-08T02:31:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/4c1f5d444c314216820ba688f0cf01e6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/4c1f5d444c314216820ba688f0cf01e6 2024-11-08T02:31:52,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/4c1f5d444c314216820ba688f0cf01e6, entries=150, sequenceid=312, filesize=12.0 K 2024-11-08T02:31:52,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0bc3c725c17ccfcde445d058bf2f374c in 866ms, sequenceid=312, compaction requested=false 2024-11-08T02:31:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-08T02:31:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-11-08T02:31:52,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-11-08T02:31:52,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-11-08T02:31:52,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 1.0210 sec 2024-11-08T02:31:52,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-08T02:31:52,713 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 83 completed 2024-11-08T02:31:52,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:31:52,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-08T02:31:52,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-08T02:31:52,715 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:31:52,715 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:31:52,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:31:52,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-08T02:31:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:52,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:31:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:52,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:52,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/89f85543ab9143bea38016534276de23 is 50, key is test_row_0/A:col10/1731033112806/Put/seqid=0 2024-11-08T02:31:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742103_1279 (size=14741) 2024-11-08T02:31:52,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033172838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033172838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033172840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033172840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,866 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:52,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:52,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:52,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:52,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:52,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:52,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033172942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033172942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033172943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:52,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033172943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:52,970 DEBUG [Thread-1019 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1730a60f to 127.0.0.1:56877 2024-11-08T02:31:52,970 DEBUG [Thread-1019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:52,971 DEBUG [Thread-1011 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0eb04aeb to 127.0.0.1:56877 2024-11-08T02:31:52,972 DEBUG [Thread-1011 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:52,972 DEBUG [Thread-1013 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a0e9c8f to 127.0.0.1:56877 2024-11-08T02:31:52,972 DEBUG [Thread-1013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:52,972 DEBUG [Thread-1017 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10e6bf6a to 127.0.0.1:56877 2024-11-08T02:31:52,972 DEBUG [Thread-1017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:52,973 DEBUG [Thread-1015 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d68f787 to 127.0.0.1:56877 2024-11-08T02:31:52,973 DEBUG [Thread-1015 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-08T02:31:53,019 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:53,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:53,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033173146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033173147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033173147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033173147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,173 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:53,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/89f85543ab9143bea38016534276de23 2024-11-08T02:31:53,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/ee848148db5546f18dadd4b037aaa990 is 50, key is test_row_0/B:col10/1731033112806/Put/seqid=0 2024-11-08T02:31:53,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742104_1280 (size=12301) 2024-11-08T02:31:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-08T02:31:53,330 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:53,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033173449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033173449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033173452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033173452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,485 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:53,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,640 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:53,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/ee848148db5546f18dadd4b037aaa990 2024-11-08T02:31:53,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/5e2bb0354cc6466b9da9b04871b4f5f7 is 50, key is test_row_0/C:col10/1731033112806/Put/seqid=0 2024-11-08T02:31:53,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742105_1281 (size=12301) 2024-11-08T02:31:53,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36196 deadline: 1731033173758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,759 DEBUG [Thread-1002 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8145 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:31:53,796 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-08T02:31:53,950 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:53,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:53,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:31:53,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36204 deadline: 1731033173955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36216 deadline: 1731033173955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731033173957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:31:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731033173958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:31:54,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/5e2bb0354cc6466b9da9b04871b4f5f7 2024-11-08T02:31:54,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/89f85543ab9143bea38016534276de23 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/89f85543ab9143bea38016534276de23 2024-11-08T02:31:54,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/89f85543ab9143bea38016534276de23, entries=200, sequenceid=334, filesize=14.4 K 2024-11-08T02:31:54,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/ee848148db5546f18dadd4b037aaa990 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ee848148db5546f18dadd4b037aaa990 2024-11-08T02:31:54,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ee848148db5546f18dadd4b037aaa990, entries=150, sequenceid=334, filesize=12.0 K 2024-11-08T02:31:54,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/5e2bb0354cc6466b9da9b04871b4f5f7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/5e2bb0354cc6466b9da9b04871b4f5f7 2024-11-08T02:31:54,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/5e2bb0354cc6466b9da9b04871b4f5f7, entries=150, sequenceid=334, filesize=12.0 K 2024-11-08T02:31:54,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0bc3c725c17ccfcde445d058bf2f374c in 1277ms, sequenceid=334, compaction requested=true 2024-11-08T02:31:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:31:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:31:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:54,106 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:54,106 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0bc3c725c17ccfcde445d058bf2f374c:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:31:54,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:54,107 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:31:54,107 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:54,107 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:54,107 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/B is initiating minor compaction (all files) 2024-11-08T02:31:54,107 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/A is initiating minor compaction (all files) 2024-11-08T02:31:54,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-08T02:31:54,107 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/B in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:54,107 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/A in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:54,107 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/281ef86a2e904dab89c6430d389d1092, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2824e274cb5249feae5e8c6d747d8a0d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ee848148db5546f18dadd4b037aaa990] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=36.7 K 2024-11-08T02:31:54,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:54,107 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/a65ef276265e42b0bcffdd3032178329, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8f9fd9b647ed480d9c7caefa7fbfb720, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/89f85543ab9143bea38016534276de23] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=39.1 K 2024-11-08T02:31:54,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:31:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:31:54,108 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 281ef86a2e904dab89c6430d389d1092, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033109427 2024-11-08T02:31:54,108 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting a65ef276265e42b0bcffdd3032178329, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033109427 2024-11-08T02:31:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:31:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:31:54,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:31:54,108 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 2824e274cb5249feae5e8c6d747d8a0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731033110048 2024-11-08T02:31:54,108 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8f9fd9b647ed480d9c7caefa7fbfb720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731033110048 2024-11-08T02:31:54,109 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting ee848148db5546f18dadd4b037aaa990, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731033112796 2024-11-08T02:31:54,109 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 89f85543ab9143bea38016534276de23, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731033112186 2024-11-08T02:31:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/c7b932e473b74a6fa091f59d31eb0661 is 50, key is test_row_0/A:col10/1731033112837/Put/seqid=0 2024-11-08T02:31:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742106_1282 (size=12301) 2024-11-08T02:31:54,117 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:54,117 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#B#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:54,118 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/9d87e4e4d8b7488bb240003e5ffdd527 is 50, key is test_row_0/B:col10/1731033112806/Put/seqid=0 2024-11-08T02:31:54,118 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/bc0abbfdf66141568fce21dd191aa04c is 50, key is test_row_0/A:col10/1731033112806/Put/seqid=0 2024-11-08T02:31:54,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742107_1283 (size=13085) 2024-11-08T02:31:54,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742108_1284 (size=13085) 2024-11-08T02:31:54,519 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/c7b932e473b74a6fa091f59d31eb0661 2024-11-08T02:31:54,528 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/9d87e4e4d8b7488bb240003e5ffdd527 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/9d87e4e4d8b7488bb240003e5ffdd527 2024-11-08T02:31:54,528 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/bc0abbfdf66141568fce21dd191aa04c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc0abbfdf66141568fce21dd191aa04c 2024-11-08T02:31:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/bd3ea1a097d94dbf86962a71e40134a2 is 50, key is test_row_0/B:col10/1731033112837/Put/seqid=0 2024-11-08T02:31:54,533 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/B of 0bc3c725c17ccfcde445d058bf2f374c into 9d87e4e4d8b7488bb240003e5ffdd527(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:54,533 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/A of 0bc3c725c17ccfcde445d058bf2f374c into bc0abbfdf66141568fce21dd191aa04c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:54,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:54,533 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:54,533 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/A, priority=13, startTime=1731033114106; duration=0sec 2024-11-08T02:31:54,533 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/B, priority=13, startTime=1731033114106; duration=0sec 2024-11-08T02:31:54,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:31:54,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:A 2024-11-08T02:31:54,533 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:54,533 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:B 2024-11-08T02:31:54,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:31:54,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742109_1285 (size=12301) 2024-11-08T02:31:54,534 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:31:54,534 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 0bc3c725c17ccfcde445d058bf2f374c/C is initiating minor compaction (all files) 2024-11-08T02:31:54,534 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 0bc3c725c17ccfcde445d058bf2f374c/C in TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:54,534 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b8ae9b961b33434c9880a51be36084b3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/4c1f5d444c314216820ba688f0cf01e6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/5e2bb0354cc6466b9da9b04871b4f5f7] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp, totalSize=36.7 K 2024-11-08T02:31:54,534 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b8ae9b961b33434c9880a51be36084b3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033109427 2024-11-08T02:31:54,534 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4c1f5d444c314216820ba688f0cf01e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731033110048 2024-11-08T02:31:54,535 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5e2bb0354cc6466b9da9b04871b4f5f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731033112796 2024-11-08T02:31:54,540 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0bc3c725c17ccfcde445d058bf2f374c#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:31:54,540 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/87c5cc1457534d6eaabc97eca0ef89c1 is 50, key is test_row_0/C:col10/1731033112806/Put/seqid=0 2024-11-08T02:31:54,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742110_1286 (size=13085) 2024-11-08T02:31:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-08T02:31:54,952 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/bd3ea1a097d94dbf86962a71e40134a2 2024-11-08T02:31:54,956 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/87c5cc1457534d6eaabc97eca0ef89c1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/87c5cc1457534d6eaabc97eca0ef89c1 2024-11-08T02:31:54,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/c25911f3b4524a18a5b63aff2e0810b6 is 50, key is test_row_0/C:col10/1731033112837/Put/seqid=0 2024-11-08T02:31:54,960 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 0bc3c725c17ccfcde445d058bf2f374c/C of 0bc3c725c17ccfcde445d058bf2f374c into 87c5cc1457534d6eaabc97eca0ef89c1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:31:54,960 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:54,960 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c., storeName=0bc3c725c17ccfcde445d058bf2f374c/C, priority=13, startTime=1731033114106; duration=0sec 2024-11-08T02:31:54,960 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:31:54,960 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0bc3c725c17ccfcde445d058bf2f374c:C 2024-11-08T02:31:54,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742111_1287 (size=12301) 2024-11-08T02:31:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:31:54,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. as already flushing 2024-11-08T02:31:54,963 DEBUG [Thread-1000 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6862e3ce to 127.0.0.1:56877 2024-11-08T02:31:54,963 DEBUG [Thread-1000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:54,965 DEBUG [Thread-1008 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x560ec309 to 127.0.0.1:56877 2024-11-08T02:31:54,965 DEBUG [Thread-1008 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:54,966 DEBUG [Thread-1004 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08d0caa5 to 127.0.0.1:56877 2024-11-08T02:31:54,966 DEBUG [Thread-1004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:54,968 DEBUG [Thread-1006 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x43f04e0e to 127.0.0.1:56877 2024-11-08T02:31:54,968 DEBUG [Thread-1006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:31:55,057 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:31:55,364 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/c25911f3b4524a18a5b63aff2e0810b6 2024-11-08T02:31:55,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/c7b932e473b74a6fa091f59d31eb0661 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/c7b932e473b74a6fa091f59d31eb0661 2024-11-08T02:31:55,378 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/c7b932e473b74a6fa091f59d31eb0661, entries=150, sequenceid=348, filesize=12.0 K 2024-11-08T02:31:55,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/bd3ea1a097d94dbf86962a71e40134a2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bd3ea1a097d94dbf86962a71e40134a2 2024-11-08T02:31:55,382 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bd3ea1a097d94dbf86962a71e40134a2, entries=150, sequenceid=348, filesize=12.0 K 2024-11-08T02:31:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/c25911f3b4524a18a5b63aff2e0810b6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c25911f3b4524a18a5b63aff2e0810b6 2024-11-08T02:31:55,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c25911f3b4524a18a5b63aff2e0810b6, entries=150, sequenceid=348, filesize=12.0 K 2024-11-08T02:31:55,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=26.84 KB/27480 for 0bc3c725c17ccfcde445d058bf2f374c in 1279ms, sequenceid=348, compaction requested=false 2024-11-08T02:31:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2538): Flush status journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:31:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:31:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-08T02:31:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-11-08T02:31:55,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-08T02:31:55,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6730 sec 2024-11-08T02:31:55,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.6760 sec 2024-11-08T02:31:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-08T02:31:56,823 INFO [Thread-1010 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 85 completed 2024-11-08T02:31:59,914 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46319,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:01,125 ERROR [LeaseRenewer:jenkins@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46319,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:03,817 DEBUG [Thread-1002 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d296fed to 127.0.0.1:56877 2024-11-08T02:32:03,817 DEBUG [Thread-1002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6023 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5922 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6004 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6018 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5924 2024-11-08T02:32:03,817 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:32:03,817 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T02:32:03,818 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a569490 to 127.0.0.1:56877 2024-11-08T02:32:03,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:03,818 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-08T02:32:03,819 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:32:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T02:32:03,822 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033123821"}]},"ts":"1731033123821"} 2024-11-08T02:32:03,823 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:32:03,881 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:32:03,883 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:32:03,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bc3c725c17ccfcde445d058bf2f374c, UNASSIGN}] 2024-11-08T02:32:03,886 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=89, ppid=88, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bc3c725c17ccfcde445d058bf2f374c, UNASSIGN 2024-11-08T02:32:03,887 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=89 updating hbase:meta row=0bc3c725c17ccfcde445d058bf2f374c, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:03,889 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:32:03,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; CloseRegionProcedure 0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:32:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T02:32:04,042 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:04,043 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] handler.UnassignRegionHandler(124): Close 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:32:04,043 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:32:04,044 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(1681): Closing 0bc3c725c17ccfcde445d058bf2f374c, disabling compactions & flushes 2024-11-08T02:32:04,044 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:32:04,044 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:32:04,044 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. after waiting 0 ms 2024-11-08T02:32:04,044 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:32:04,044 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(2837): Flushing 0bc3c725c17ccfcde445d058bf2f374c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T02:32:04,045 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=A 2024-11-08T02:32:04,045 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:04,045 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=B 2024-11-08T02:32:04,046 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:04,046 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0bc3c725c17ccfcde445d058bf2f374c, store=C 2024-11-08T02:32:04,046 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:04,052 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/47f04e1602434ead81d08e2395c2ce30 is 50, key is test_row_0/A:col10/1731033114967/Put/seqid=0 2024-11-08T02:32:04,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742112_1288 (size=9857) 2024-11-08T02:32:04,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T02:32:04,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T02:32:04,457 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/47f04e1602434ead81d08e2395c2ce30 2024-11-08T02:32:04,470 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/5e3aea28611d44d4ba0249f17976a407 is 50, key is test_row_0/B:col10/1731033114967/Put/seqid=0 2024-11-08T02:32:04,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742113_1289 (size=9857) 2024-11-08T02:32:04,875 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/5e3aea28611d44d4ba0249f17976a407 2024-11-08T02:32:04,886 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d6a468abd55c47fa8eaacd81803f04eb is 50, key is test_row_0/C:col10/1731033114967/Put/seqid=0 2024-11-08T02:32:04,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742114_1290 (size=9857) 2024-11-08T02:32:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T02:32:05,293 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d6a468abd55c47fa8eaacd81803f04eb 2024-11-08T02:32:05,303 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/A/47f04e1602434ead81d08e2395c2ce30 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/47f04e1602434ead81d08e2395c2ce30 2024-11-08T02:32:05,308 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/47f04e1602434ead81d08e2395c2ce30, entries=100, sequenceid=359, filesize=9.6 K 2024-11-08T02:32:05,309 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/B/5e3aea28611d44d4ba0249f17976a407 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5e3aea28611d44d4ba0249f17976a407 2024-11-08T02:32:05,313 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5e3aea28611d44d4ba0249f17976a407, entries=100, sequenceid=359, filesize=9.6 K 2024-11-08T02:32:05,313 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/.tmp/C/d6a468abd55c47fa8eaacd81803f04eb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d6a468abd55c47fa8eaacd81803f04eb 2024-11-08T02:32:05,317 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d6a468abd55c47fa8eaacd81803f04eb, entries=100, sequenceid=359, filesize=9.6 K 2024-11-08T02:32:05,318 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0bc3c725c17ccfcde445d058bf2f374c in 1273ms, sequenceid=359, compaction requested=true 2024-11-08T02:32:05,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/de1d659c20434864a05a77ebf67e8796, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3e345698c4d74231ade6b69a981a9c07, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/ef7fd67a83e04f00868ec4af88f79b36, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/07ddc29c9360472daeddcae5281889e1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/868c2b38ff09417d9fa6ec62c5f1642b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/1200a2d4372840adb10e9a49fbcfe65b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc3975a95d2b4bfe8d1712ba2f952340, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/fc346501fc27487a96ec3366564f90bb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/941faf9f04c8458ba0494b9c38a40021, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/b7b597f98a5c4b39814e0fe4f0c29a62, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/190fe7a4fe3842cd9e5027c614995e99, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/9f157a071ee84332864816e81c1106b4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/33a3be06012c4ddd9c88a7c5af79a978, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bea90774e97a49eca3959cb4c6c3b11f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8e4f6ba0c45b427088b414a6d47284be, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/95611e302dcf4c579ee1acf981f55fd7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3d647ee772404e6ca64679106030f6cf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/08671855036a4f9aa7493b5d70bc3fa5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/31e2cf6a830f4499ae118ec63e4c8416, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/197583de9a724f45a1a23602c3743aba, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/a65ef276265e42b0bcffdd3032178329, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8f9fd9b647ed480d9c7caefa7fbfb720, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/89f85543ab9143bea38016534276de23] to archive 2024-11-08T02:32:05,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:05,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/de1d659c20434864a05a77ebf67e8796 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/de1d659c20434864a05a77ebf67e8796 2024-11-08T02:32:05,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3e345698c4d74231ade6b69a981a9c07 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3e345698c4d74231ade6b69a981a9c07 2024-11-08T02:32:05,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/ef7fd67a83e04f00868ec4af88f79b36 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/ef7fd67a83e04f00868ec4af88f79b36 2024-11-08T02:32:05,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/07ddc29c9360472daeddcae5281889e1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/07ddc29c9360472daeddcae5281889e1 2024-11-08T02:32:05,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/868c2b38ff09417d9fa6ec62c5f1642b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/868c2b38ff09417d9fa6ec62c5f1642b 2024-11-08T02:32:05,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/1200a2d4372840adb10e9a49fbcfe65b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/1200a2d4372840adb10e9a49fbcfe65b 2024-11-08T02:32:05,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc3975a95d2b4bfe8d1712ba2f952340 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc3975a95d2b4bfe8d1712ba2f952340 2024-11-08T02:32:05,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/fc346501fc27487a96ec3366564f90bb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/fc346501fc27487a96ec3366564f90bb 2024-11-08T02:32:05,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/941faf9f04c8458ba0494b9c38a40021 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/941faf9f04c8458ba0494b9c38a40021 2024-11-08T02:32:05,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/b7b597f98a5c4b39814e0fe4f0c29a62 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/b7b597f98a5c4b39814e0fe4f0c29a62 2024-11-08T02:32:05,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/190fe7a4fe3842cd9e5027c614995e99 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/190fe7a4fe3842cd9e5027c614995e99 2024-11-08T02:32:05,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/9f157a071ee84332864816e81c1106b4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/9f157a071ee84332864816e81c1106b4 2024-11-08T02:32:05,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/33a3be06012c4ddd9c88a7c5af79a978 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/33a3be06012c4ddd9c88a7c5af79a978 2024-11-08T02:32:05,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bea90774e97a49eca3959cb4c6c3b11f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bea90774e97a49eca3959cb4c6c3b11f 2024-11-08T02:32:05,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8e4f6ba0c45b427088b414a6d47284be to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8e4f6ba0c45b427088b414a6d47284be 2024-11-08T02:32:05,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/95611e302dcf4c579ee1acf981f55fd7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/95611e302dcf4c579ee1acf981f55fd7 2024-11-08T02:32:05,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3d647ee772404e6ca64679106030f6cf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/3d647ee772404e6ca64679106030f6cf 2024-11-08T02:32:05,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/08671855036a4f9aa7493b5d70bc3fa5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/08671855036a4f9aa7493b5d70bc3fa5 2024-11-08T02:32:05,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/31e2cf6a830f4499ae118ec63e4c8416 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/31e2cf6a830f4499ae118ec63e4c8416 2024-11-08T02:32:05,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/197583de9a724f45a1a23602c3743aba to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/197583de9a724f45a1a23602c3743aba 2024-11-08T02:32:05,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/a65ef276265e42b0bcffdd3032178329 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/a65ef276265e42b0bcffdd3032178329 2024-11-08T02:32:05,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8f9fd9b647ed480d9c7caefa7fbfb720 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/8f9fd9b647ed480d9c7caefa7fbfb720 2024-11-08T02:32:05,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/89f85543ab9143bea38016534276de23 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/89f85543ab9143bea38016534276de23 2024-11-08T02:32:05,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/87342d5417314921b4106a7519e997e7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/185f4ea4e1a84752b7f3df568f5cf852, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3ceb36dd06ec474fbef2ad1a31ab8ebf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bbea71a5631c4371b798cafd54ee090c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/689808a987d84906a51121115697d697, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2ef298a8a3b04b70afd68185d90dd8dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/33313dfa0f064d869bae5c23ad727457, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/b75bd3b713d547eeb29a1847315959a8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1e61576645014707b062967c220138ad, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3575a6b646d54a258d30deb8afbc4d4a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/723f7debeef449c0b2692fd2ef10ab1a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1d02000ed8584647b16e1a1c312cfc6c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/064511b06ba145c597c70d9e23005b51, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2d0a5226720e4b8498bdb29cb23080e2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5b87fcabcd8d4edead6f95c49b67be07, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ce044f11cc47474a899f363b0345a375, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/91413fad377843c7bae9e6609e896dce, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/43bdad14cde14427bc0521b7e1e48279, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/08a153b5a7e04f9ebaf34b430489bcdf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/281ef86a2e904dab89c6430d389d1092, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/90aa40af910a412c9d672d4084c0e5eb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2824e274cb5249feae5e8c6d747d8a0d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ee848148db5546f18dadd4b037aaa990] to archive 2024-11-08T02:32:05,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:05,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/87342d5417314921b4106a7519e997e7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/87342d5417314921b4106a7519e997e7 2024-11-08T02:32:05,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/185f4ea4e1a84752b7f3df568f5cf852 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/185f4ea4e1a84752b7f3df568f5cf852 2024-11-08T02:32:05,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3ceb36dd06ec474fbef2ad1a31ab8ebf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3ceb36dd06ec474fbef2ad1a31ab8ebf 2024-11-08T02:32:05,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bbea71a5631c4371b798cafd54ee090c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bbea71a5631c4371b798cafd54ee090c 2024-11-08T02:32:05,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/689808a987d84906a51121115697d697 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/689808a987d84906a51121115697d697 2024-11-08T02:32:05,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2ef298a8a3b04b70afd68185d90dd8dc to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2ef298a8a3b04b70afd68185d90dd8dc 2024-11-08T02:32:05,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/33313dfa0f064d869bae5c23ad727457 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/33313dfa0f064d869bae5c23ad727457 2024-11-08T02:32:05,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/b75bd3b713d547eeb29a1847315959a8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/b75bd3b713d547eeb29a1847315959a8 2024-11-08T02:32:05,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1e61576645014707b062967c220138ad to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1e61576645014707b062967c220138ad 2024-11-08T02:32:05,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3575a6b646d54a258d30deb8afbc4d4a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/3575a6b646d54a258d30deb8afbc4d4a 2024-11-08T02:32:05,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/723f7debeef449c0b2692fd2ef10ab1a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/723f7debeef449c0b2692fd2ef10ab1a 2024-11-08T02:32:05,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1d02000ed8584647b16e1a1c312cfc6c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/1d02000ed8584647b16e1a1c312cfc6c 2024-11-08T02:32:05,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/064511b06ba145c597c70d9e23005b51 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/064511b06ba145c597c70d9e23005b51 2024-11-08T02:32:05,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2d0a5226720e4b8498bdb29cb23080e2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2d0a5226720e4b8498bdb29cb23080e2 2024-11-08T02:32:05,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5b87fcabcd8d4edead6f95c49b67be07 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5b87fcabcd8d4edead6f95c49b67be07 2024-11-08T02:32:05,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ce044f11cc47474a899f363b0345a375 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ce044f11cc47474a899f363b0345a375 2024-11-08T02:32:05,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/91413fad377843c7bae9e6609e896dce to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/91413fad377843c7bae9e6609e896dce 2024-11-08T02:32:05,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/43bdad14cde14427bc0521b7e1e48279 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/43bdad14cde14427bc0521b7e1e48279 2024-11-08T02:32:05,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/08a153b5a7e04f9ebaf34b430489bcdf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/08a153b5a7e04f9ebaf34b430489bcdf 2024-11-08T02:32:05,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/281ef86a2e904dab89c6430d389d1092 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/281ef86a2e904dab89c6430d389d1092 2024-11-08T02:32:05,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/90aa40af910a412c9d672d4084c0e5eb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/90aa40af910a412c9d672d4084c0e5eb 2024-11-08T02:32:05,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2824e274cb5249feae5e8c6d747d8a0d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/2824e274cb5249feae5e8c6d747d8a0d 2024-11-08T02:32:05,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ee848148db5546f18dadd4b037aaa990 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/ee848148db5546f18dadd4b037aaa990 2024-11-08T02:32:05,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ee5c72060d6f46daab1353cb2f06a91b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/55a6ea6c5c2a4ed6b975c8f7f39008b1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/71567d33f9ce4434afa4d1648e45f5d0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/854cf9f83123457f8a10e3845cf90065, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/444111b4b06e477eab3e329ffb0161ac, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/e45c8e0d140d487588504a5641682f7b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d070676c4c5e442cb54e36238de9d12d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d0a22201d33a466eacf018e6b73b61d4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/8791ed7ed83e42b6847499aa86672e43, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f0b868a9025d478989925c8e1675c9a6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/dac81e6bd0d04f1ba44c371e7fef4964, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/a43cd558a0214443bdcb15bb367b122a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ec6b76a8af4845afb22e0888f8b40ef3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/921343e7a90b4c45862912820e3f83a4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f9d2b368966a41ab92a9102c141551fe, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/fcfe1a968cf948bcac21806ccd6c9faa, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c8c9ae6b894b49c39cba935bfd791505, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b91fcb458f1c447291ad3a07bc0ac75d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d7e2d2b294864a7a8f871a9a72261be7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b8ae9b961b33434c9880a51be36084b3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/7a64afc548ff4a13b770830c603cc493, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/4c1f5d444c314216820ba688f0cf01e6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/5e2bb0354cc6466b9da9b04871b4f5f7] to archive 2024-11-08T02:32:05,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:05,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ee5c72060d6f46daab1353cb2f06a91b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ee5c72060d6f46daab1353cb2f06a91b 2024-11-08T02:32:05,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/55a6ea6c5c2a4ed6b975c8f7f39008b1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/55a6ea6c5c2a4ed6b975c8f7f39008b1 2024-11-08T02:32:05,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/71567d33f9ce4434afa4d1648e45f5d0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/71567d33f9ce4434afa4d1648e45f5d0 2024-11-08T02:32:05,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/854cf9f83123457f8a10e3845cf90065 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/854cf9f83123457f8a10e3845cf90065 2024-11-08T02:32:05,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/444111b4b06e477eab3e329ffb0161ac to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/444111b4b06e477eab3e329ffb0161ac 2024-11-08T02:32:05,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/e45c8e0d140d487588504a5641682f7b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/e45c8e0d140d487588504a5641682f7b 2024-11-08T02:32:05,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d070676c4c5e442cb54e36238de9d12d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d070676c4c5e442cb54e36238de9d12d 2024-11-08T02:32:05,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d0a22201d33a466eacf018e6b73b61d4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d0a22201d33a466eacf018e6b73b61d4 2024-11-08T02:32:05,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/8791ed7ed83e42b6847499aa86672e43 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/8791ed7ed83e42b6847499aa86672e43 2024-11-08T02:32:05,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f0b868a9025d478989925c8e1675c9a6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f0b868a9025d478989925c8e1675c9a6 2024-11-08T02:32:05,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/dac81e6bd0d04f1ba44c371e7fef4964 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/dac81e6bd0d04f1ba44c371e7fef4964 2024-11-08T02:32:05,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/a43cd558a0214443bdcb15bb367b122a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/a43cd558a0214443bdcb15bb367b122a 2024-11-08T02:32:05,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ec6b76a8af4845afb22e0888f8b40ef3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/ec6b76a8af4845afb22e0888f8b40ef3 2024-11-08T02:32:05,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/921343e7a90b4c45862912820e3f83a4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/921343e7a90b4c45862912820e3f83a4 2024-11-08T02:32:05,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f9d2b368966a41ab92a9102c141551fe to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/f9d2b368966a41ab92a9102c141551fe 2024-11-08T02:32:05,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/fcfe1a968cf948bcac21806ccd6c9faa to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/fcfe1a968cf948bcac21806ccd6c9faa 2024-11-08T02:32:05,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c8c9ae6b894b49c39cba935bfd791505 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c8c9ae6b894b49c39cba935bfd791505 2024-11-08T02:32:05,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b91fcb458f1c447291ad3a07bc0ac75d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b91fcb458f1c447291ad3a07bc0ac75d 2024-11-08T02:32:05,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d7e2d2b294864a7a8f871a9a72261be7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d7e2d2b294864a7a8f871a9a72261be7 2024-11-08T02:32:05,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b8ae9b961b33434c9880a51be36084b3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/b8ae9b961b33434c9880a51be36084b3 2024-11-08T02:32:05,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/7a64afc548ff4a13b770830c603cc493 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/7a64afc548ff4a13b770830c603cc493 2024-11-08T02:32:05,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/4c1f5d444c314216820ba688f0cf01e6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/4c1f5d444c314216820ba688f0cf01e6 2024-11-08T02:32:05,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/5e2bb0354cc6466b9da9b04871b4f5f7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/5e2bb0354cc6466b9da9b04871b4f5f7 2024-11-08T02:32:05,422 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/recovered.edits/362.seqid, newMaxSeqId=362, maxSeqId=1 2024-11-08T02:32:05,422 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c. 2024-11-08T02:32:05,422 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] regionserver.HRegion(1635): Region close journal for 0bc3c725c17ccfcde445d058bf2f374c: 2024-11-08T02:32:05,423 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=90}] handler.UnassignRegionHandler(170): Closed 0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:32:05,424 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=89 updating hbase:meta row=0bc3c725c17ccfcde445d058bf2f374c, regionState=CLOSED 2024-11-08T02:32:05,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-11-08T02:32:05,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; CloseRegionProcedure 0bc3c725c17ccfcde445d058bf2f374c, server=ecdbff65922a,40577,1731033026611 in 1.5350 sec 2024-11-08T02:32:05,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=89, resume processing ppid=88 2024-11-08T02:32:05,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, ppid=88, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=0bc3c725c17ccfcde445d058bf2f374c, UNASSIGN in 1.5400 sec 2024-11-08T02:32:05,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-08T02:32:05,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5440 sec 2024-11-08T02:32:05,428 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033125428"}]},"ts":"1731033125428"} 2024-11-08T02:32:05,429 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:32:05,469 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:32:05,472 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.6510 sec 2024-11-08T02:32:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-08T02:32:05,929 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 87 completed 2024-11-08T02:32:05,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:32:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:05,935 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=91, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:05,936 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=91, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-08T02:32:05,940 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:32:05,943 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/recovered.edits] 2024-11-08T02:32:05,946 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/47f04e1602434ead81d08e2395c2ce30 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/47f04e1602434ead81d08e2395c2ce30 2024-11-08T02:32:05,948 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc0abbfdf66141568fce21dd191aa04c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/bc0abbfdf66141568fce21dd191aa04c 2024-11-08T02:32:05,949 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/c7b932e473b74a6fa091f59d31eb0661 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/A/c7b932e473b74a6fa091f59d31eb0661 2024-11-08T02:32:05,952 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5e3aea28611d44d4ba0249f17976a407 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/5e3aea28611d44d4ba0249f17976a407 2024-11-08T02:32:05,953 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/9d87e4e4d8b7488bb240003e5ffdd527 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/9d87e4e4d8b7488bb240003e5ffdd527 2024-11-08T02:32:05,954 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bd3ea1a097d94dbf86962a71e40134a2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/B/bd3ea1a097d94dbf86962a71e40134a2 2024-11-08T02:32:05,955 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/87c5cc1457534d6eaabc97eca0ef89c1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/87c5cc1457534d6eaabc97eca0ef89c1 2024-11-08T02:32:05,956 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c25911f3b4524a18a5b63aff2e0810b6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/c25911f3b4524a18a5b63aff2e0810b6 2024-11-08T02:32:05,957 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d6a468abd55c47fa8eaacd81803f04eb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/C/d6a468abd55c47fa8eaacd81803f04eb 2024-11-08T02:32:05,959 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/recovered.edits/362.seqid to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c/recovered.edits/362.seqid 2024-11-08T02:32:05,959 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/0bc3c725c17ccfcde445d058bf2f374c 2024-11-08T02:32:05,959 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:32:05,961 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=91, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:05,965 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:32:05,967 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:32:05,968 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=91, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:05,968 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:32:05,968 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731033125968"}]},"ts":"9223372036854775807"} 2024-11-08T02:32:05,970 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-08T02:32:05,970 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 0bc3c725c17ccfcde445d058bf2f374c, NAME => 'TestAcidGuarantees,,1731033090701.0bc3c725c17ccfcde445d058bf2f374c.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:32:05,970 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:32:05,970 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731033125970"}]},"ts":"9223372036854775807"} 2024-11-08T02:32:05,972 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-08T02:32:06,011 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=91, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:06,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-08T02:32:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-08T02:32:06,038 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 91 completed 2024-11-08T02:32:06,049 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testGetAtomicity Thread=236 (was 239), OpenFileDescriptor=444 (was 456), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 235) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2159 (was 2223) 2024-11-08T02:32:06,060 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMobScanAtomicity Thread=236, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=2158 2024-11-08T02:32:06,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:32:06,062 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:32:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=92, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:06,063 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=92, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:32:06,063 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:06,063 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 92 2024-11-08T02:32:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=92 2024-11-08T02:32:06,064 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=92, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:32:06,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742115_1291 (size=960) 2024-11-08T02:32:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=92 2024-11-08T02:32:06,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T02:32:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=92 2024-11-08T02:32:06,474 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:32:06,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742116_1292 (size=53) 2024-11-08T02:32:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=92 2024-11-08T02:32:06,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:32:06,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing a2c847edb50c5b92217fdd99390ba5a5, disabling compactions & flushes 2024-11-08T02:32:06,885 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:06,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:06,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. after waiting 0 ms 2024-11-08T02:32:06,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:06,885 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:06,885 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:06,911 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=92, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:32:06,912 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731033126911"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731033126911"}]},"ts":"1731033126911"} 2024-11-08T02:32:06,914 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T02:32:06,916 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=92, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:32:06,916 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033126916"}]},"ts":"1731033126916"} 2024-11-08T02:32:06,918 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:32:06,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, ASSIGN}] 2024-11-08T02:32:06,964 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=93, ppid=92, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, ASSIGN 2024-11-08T02:32:06,966 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=93, ppid=92, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, ASSIGN; state=OFFLINE, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=false 2024-11-08T02:32:07,116 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=93 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:07,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; OpenRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:32:07,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=92 2024-11-08T02:32:07,270 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:07,276 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:07,277 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7285): Opening region: {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:32:07,277 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,277 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:32:07,278 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7327): checking encryption for a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,278 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7330): checking classloading for a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,280 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,282 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:07,283 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c847edb50c5b92217fdd99390ba5a5 columnFamilyName A 2024-11-08T02:32:07,283 DEBUG [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:07,284 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(327): Store=a2c847edb50c5b92217fdd99390ba5a5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:07,284 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,285 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:07,286 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c847edb50c5b92217fdd99390ba5a5 columnFamilyName B 2024-11-08T02:32:07,286 DEBUG [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:07,286 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(327): Store=a2c847edb50c5b92217fdd99390ba5a5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:07,286 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,288 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:07,288 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c847edb50c5b92217fdd99390ba5a5 columnFamilyName C 2024-11-08T02:32:07,288 DEBUG [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:07,289 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(327): Store=a2c847edb50c5b92217fdd99390ba5a5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:07,289 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:07,290 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,290 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,292 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:32:07,293 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1085): writing seq id for a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:07,295 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:32:07,296 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1102): Opened a2c847edb50c5b92217fdd99390ba5a5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73602747, jitterRate=0.096766397356987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:32:07,296 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1001): Region open journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:07,297 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., pid=94, masterSystemTime=1731033127270 2024-11-08T02:32:07,298 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:07,298 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:07,298 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=93 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:07,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=94, resume processing ppid=93 2024-11-08T02:32:07,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=94, ppid=93, state=SUCCESS; OpenRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 in 182 msec 2024-11-08T02:32:07,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=93, resume processing ppid=92 2024-11-08T02:32:07,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=93, ppid=92, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, ASSIGN in 339 msec 2024-11-08T02:32:07,301 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=92, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:32:07,302 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033127301"}]},"ts":"1731033127301"} 2024-11-08T02:32:07,302 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:32:07,345 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=92, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:32:07,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2830 sec 2024-11-08T02:32:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=92 2024-11-08T02:32:08,176 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 92 completed 2024-11-08T02:32:08,179 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b82ba2a to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3637e4c6 2024-11-08T02:32:08,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f7d511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:08,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:08,228 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52670, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:08,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:32:08,232 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59348, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:32:08,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:32:08,235 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:32:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=95, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:08,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742117_1293 (size=996) 2024-11-08T02:32:08,650 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.960 2024-11-08T02:32:08,650 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.996 2024-11-08T02:32:08,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:32:08,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, REOPEN/MOVE}] 2024-11-08T02:32:08,660 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=97, ppid=96, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, REOPEN/MOVE 2024-11-08T02:32:08,660 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=97 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:08,661 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:32:08,661 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; CloseRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:32:08,813 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:08,814 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] handler.UnassignRegionHandler(124): Close a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:08,814 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:32:08,814 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegion(1681): Closing a2c847edb50c5b92217fdd99390ba5a5, disabling compactions & flushes 2024-11-08T02:32:08,814 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:08,814 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:08,814 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. after waiting 0 ms 2024-11-08T02:32:08,814 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:08,822 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T02:32:08,823 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:08,823 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegion(1635): Region close journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:08,823 WARN [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] regionserver.HRegionServer(3786): Not adding moved region record: a2c847edb50c5b92217fdd99390ba5a5 to self. 2024-11-08T02:32:08,826 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=98}] handler.UnassignRegionHandler(170): Closed a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:08,826 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=97 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=CLOSED 2024-11-08T02:32:08,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97 2024-11-08T02:32:08,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; CloseRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 in 167 msec 2024-11-08T02:32:08,833 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=97, ppid=96, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, REOPEN/MOVE; state=CLOSED, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=true 2024-11-08T02:32:08,984 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=97 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:08,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=99, ppid=97, state=RUNNABLE; OpenRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:32:09,139 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,146 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,147 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7285): Opening region: {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:32:09,148 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,149 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:32:09,149 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7327): checking encryption for a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,149 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7330): checking classloading for a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,152 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,155 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:09,155 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c847edb50c5b92217fdd99390ba5a5 columnFamilyName A 2024-11-08T02:32:09,156 DEBUG [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:09,157 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(327): Store=a2c847edb50c5b92217fdd99390ba5a5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:09,157 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,158 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:09,158 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c847edb50c5b92217fdd99390ba5a5 columnFamilyName B 2024-11-08T02:32:09,158 DEBUG [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:09,158 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(327): Store=a2c847edb50c5b92217fdd99390ba5a5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:09,158 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,159 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:09,159 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2c847edb50c5b92217fdd99390ba5a5 columnFamilyName C 2024-11-08T02:32:09,159 DEBUG [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:09,160 INFO [StoreOpener-a2c847edb50c5b92217fdd99390ba5a5-1 {}] regionserver.HStore(327): Store=a2c847edb50c5b92217fdd99390ba5a5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:09,160 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,161 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,162 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,163 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:32:09,165 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1085): writing seq id for a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,166 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1102): Opened a2c847edb50c5b92217fdd99390ba5a5; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59668911, jitterRate=-0.11086393892765045}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:32:09,166 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1001): Region open journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:09,167 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., pid=99, masterSystemTime=1731033129139 2024-11-08T02:32:09,169 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,169 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,169 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=97 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=OPEN, openSeqNum=5, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=99, resume processing ppid=97 2024-11-08T02:32:09,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, ppid=97, state=SUCCESS; OpenRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 in 184 msec 2024-11-08T02:32:09,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=97, resume processing ppid=96 2024-11-08T02:32:09,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, ppid=96, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, REOPEN/MOVE in 513 msec 2024-11-08T02:32:09,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-11-08T02:32:09,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-08T02:32:09,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 941 msec 2024-11-08T02:32:09,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-08T02:32:09,181 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b6cf8cb to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72f422b4 2024-11-08T02:32:09,233 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc42ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,234 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ec15031 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2df33cdf 2024-11-08T02:32:09,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117e86d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,249 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3dd5b441 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9f472e0 2024-11-08T02:32:09,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd96549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,258 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c336ea4 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@167a78b0 2024-11-08T02:32:09,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aea41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,394 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f94d721 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5aee939b 2024-11-08T02:32:09,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e247aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,410 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f49665c to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2205f666 2024-11-08T02:32:09,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27539bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,425 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x683f8469 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6584e9ce 2024-11-08T02:32:09,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e3203d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,438 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75e4d3d0 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37ec8e3b 2024-11-08T02:32:09,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798e7fd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,450 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b308f62 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@787e5169 2024-11-08T02:32:09,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7284f16d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,463 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68035c67 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@627cad17 2024-11-08T02:32:09,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a637ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:09,479 DEBUG [hconnection-0x4fece3f0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,479 DEBUG [hconnection-0x22cef50b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,480 DEBUG [hconnection-0x11608090-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,481 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52680, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,481 DEBUG [hconnection-0x11eda4f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,481 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52696, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,481 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52704, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,482 DEBUG [hconnection-0x46311e1d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,482 DEBUG [hconnection-0x30cddfea-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,482 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52708, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,483 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:09,483 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52714, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,483 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52720, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,483 DEBUG [hconnection-0x18ebe7c7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=100, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=100, table=TestAcidGuarantees 2024-11-08T02:32:09,484 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=100, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=100, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:09,485 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52736, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,485 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=100, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=100, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:09,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=100 2024-11-08T02:32:09,486 DEBUG [hconnection-0x22ee8b05-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,486 DEBUG [hconnection-0x3366738c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,487 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52750, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,487 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52742, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,489 DEBUG [hconnection-0x2c478c20-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:09,490 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52752, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:32:09,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:09,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:09,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:09,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:09,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:09,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:09,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033189508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033189508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033189507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033189509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033189509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f1e96d9ff7144511b7d0b6e86cbe764b_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033129490/Put/seqid=0 2024-11-08T02:32:09,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742118_1294 (size=12154) 2024-11-08T02:32:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=100 2024-11-08T02:32:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033189610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033189610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033189610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033189610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033189610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,636 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:09,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:09,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=100 2024-11-08T02:32:09,789 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:09,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:09,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033189815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033189815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033189816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033189816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033189817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:09,931 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f1e96d9ff7144511b7d0b6e86cbe764b_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f1e96d9ff7144511b7d0b6e86cbe764b_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:09,931 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/ba3c0ffef91f4ba281f1898d4368c35c, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:09,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/ba3c0ffef91f4ba281f1898d4368c35c is 175, key is test_row_0/A:col10/1731033129490/Put/seqid=0 2024-11-08T02:32:09,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742119_1295 (size=30955) 2024-11-08T02:32:09,941 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:09,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:09,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:09,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=100 2024-11-08T02:32:10,093 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033190121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033190122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033190122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033190122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033190123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,246 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/ba3c0ffef91f4ba281f1898d4368c35c 2024-11-08T02:32:10,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/32b112df40954f27969d6a09d4251137 is 50, key is test_row_0/B:col10/1731033129490/Put/seqid=0 2024-11-08T02:32:10,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742120_1296 (size=12001) 2024-11-08T02:32:10,398 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:10,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,550 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=100 2024-11-08T02:32:10,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033190624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033190626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033190627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033190627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033190629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,703 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:10,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:10,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/32b112df40954f27969d6a09d4251137 2024-11-08T02:32:10,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a6136c2e77534c25987f0ef92e152004 is 50, key is test_row_0/C:col10/1731033129490/Put/seqid=0 2024-11-08T02:32:10,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742121_1297 (size=12001) 2024-11-08T02:32:10,855 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:10,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:10,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:10,949 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:32:11,007 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:11,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:11,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:11,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:11,160 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:11,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] handler.RSProcedureHandler(58): pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=101 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=101 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:11,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a6136c2e77534c25987f0ef92e152004 2024-11-08T02:32:11,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/ba3c0ffef91f4ba281f1898d4368c35c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ba3c0ffef91f4ba281f1898d4368c35c 2024-11-08T02:32:11,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ba3c0ffef91f4ba281f1898d4368c35c, entries=150, sequenceid=18, filesize=30.2 K 2024-11-08T02:32:11,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/32b112df40954f27969d6a09d4251137 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32b112df40954f27969d6a09d4251137 2024-11-08T02:32:11,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32b112df40954f27969d6a09d4251137, entries=150, sequenceid=18, filesize=11.7 K 2024-11-08T02:32:11,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a6136c2e77534c25987f0ef92e152004 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a6136c2e77534c25987f0ef92e152004 2024-11-08T02:32:11,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a6136c2e77534c25987f0ef92e152004, entries=150, sequenceid=18, filesize=11.7 K 2024-11-08T02:32:11,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a2c847edb50c5b92217fdd99390ba5a5 in 1721ms, sequenceid=18, compaction requested=false 2024-11-08T02:32:11,213 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T02:32:11,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:11,312 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=101 2024-11-08T02:32:11,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:11,312 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:32:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:11,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:11,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080fb45b3acfa14148b9b8e2ad0451f85d_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033129507/Put/seqid=0 2024-11-08T02:32:11,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742122_1298 (size=12154) 2024-11-08T02:32:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=100 2024-11-08T02:32:11,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:11,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033191641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033191642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033191642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033191642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033191644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:11,725 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080fb45b3acfa14148b9b8e2ad0451f85d_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080fb45b3acfa14148b9b8e2ad0451f85d_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:11,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/79a9db9f80d14bbab6d1704765f5aef2, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:11,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/79a9db9f80d14bbab6d1704765f5aef2 is 175, key is test_row_0/A:col10/1731033129507/Put/seqid=0 2024-11-08T02:32:11,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742123_1299 (size=30955) 2024-11-08T02:32:11,741 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/79a9db9f80d14bbab6d1704765f5aef2 2024-11-08T02:32:11,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/324a631ddfc5463ca382393065ead094 is 50, key is test_row_0/B:col10/1731033129507/Put/seqid=0 2024-11-08T02:32:11,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033191745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033191747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033191748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033191748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742124_1300 (size=12001) 2024-11-08T02:32:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033191950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033191950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033191952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:11,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:11,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033191952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,151 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/324a631ddfc5463ca382393065ead094 2024-11-08T02:32:12,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a35546371bc04e3c991961023616d879 is 50, key is test_row_0/C:col10/1731033129507/Put/seqid=0 2024-11-08T02:32:12,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742125_1301 (size=12001) 2024-11-08T02:32:12,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033192255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033192255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033192257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033192257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,565 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a35546371bc04e3c991961023616d879 2024-11-08T02:32:12,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/79a9db9f80d14bbab6d1704765f5aef2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/79a9db9f80d14bbab6d1704765f5aef2 2024-11-08T02:32:12,573 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/79a9db9f80d14bbab6d1704765f5aef2, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T02:32:12,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/324a631ddfc5463ca382393065ead094 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/324a631ddfc5463ca382393065ead094 2024-11-08T02:32:12,582 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/324a631ddfc5463ca382393065ead094, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:32:12,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a35546371bc04e3c991961023616d879 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a35546371bc04e3c991961023616d879 2024-11-08T02:32:12,588 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a35546371bc04e3c991961023616d879, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:32:12,589 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for a2c847edb50c5b92217fdd99390ba5a5 in 1277ms, sequenceid=40, compaction requested=false 2024-11-08T02:32:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:12,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=101}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=101 2024-11-08T02:32:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=101 2024-11-08T02:32:12,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=101, resume processing ppid=100 2024-11-08T02:32:12,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=101, ppid=100, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1050 sec 2024-11-08T02:32:12,592 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=100, table=TestAcidGuarantees in 3.1080 sec 2024-11-08T02:32:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:12,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:32:12,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:12,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:12,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:12,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:12,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:12,765 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:12,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110803cdbd01c1354620b41350cd14e0ad20_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033132762/Put/seqid=0 2024-11-08T02:32:12,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742126_1302 (size=14594) 2024-11-08T02:32:12,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033192791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033192792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033192792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033192796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033192897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033192899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033192901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:12,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:12,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033192901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033193101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033193104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033193106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033193106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,176 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:13,179 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110803cdbd01c1354620b41350cd14e0ad20_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110803cdbd01c1354620b41350cd14e0ad20_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:13,180 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/c377d117ffb4429182fa0c914acd5157, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:13,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/c377d117ffb4429182fa0c914acd5157 is 175, key is test_row_0/A:col10/1731033132762/Put/seqid=0 2024-11-08T02:32:13,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742127_1303 (size=39549) 2024-11-08T02:32:13,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033193406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033193410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033193410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033193412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/c377d117ffb4429182fa0c914acd5157 2024-11-08T02:32:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=100 2024-11-08T02:32:13,589 INFO [Thread-1364 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 100 completed 2024-11-08T02:32:13,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees 2024-11-08T02:32:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-08T02:32:13,592 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=102, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:13,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/6db780d16d3446509ec5b3b5a6936575 is 50, key is test_row_0/B:col10/1731033132762/Put/seqid=0 2024-11-08T02:32:13,593 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=102, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:13,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:13,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742128_1304 (size=12001) 2024-11-08T02:32:13,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033193653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,658 DEBUG [Thread-1356 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4149 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-08T02:32:13,745 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-08T02:32:13,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:13,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:13,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:13,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:13,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:13,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:13,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-08T02:32:13,897 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-08T02:32:13,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:13,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:13,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:13,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:13,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:13,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033193913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033193914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033193915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:13,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033193919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/6db780d16d3446509ec5b3b5a6936575 2024-11-08T02:32:14,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/d8816685a1fe4dbc908726c4fd946aa2 is 50, key is test_row_0/C:col10/1731033132762/Put/seqid=0 2024-11-08T02:32:14,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742129_1305 (size=12001) 2024-11-08T02:32:14,049 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-08T02:32:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-08T02:32:14,202 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-08T02:32:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,354 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-08T02:32:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:14,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/d8816685a1fe4dbc908726c4fd946aa2 2024-11-08T02:32:14,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/c377d117ffb4429182fa0c914acd5157 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c377d117ffb4429182fa0c914acd5157 2024-11-08T02:32:14,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c377d117ffb4429182fa0c914acd5157, entries=200, sequenceid=56, filesize=38.6 K 2024-11-08T02:32:14,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/6db780d16d3446509ec5b3b5a6936575 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/6db780d16d3446509ec5b3b5a6936575 2024-11-08T02:32:14,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/6db780d16d3446509ec5b3b5a6936575, entries=150, sequenceid=56, filesize=11.7 K 2024-11-08T02:32:14,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/d8816685a1fe4dbc908726c4fd946aa2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d8816685a1fe4dbc908726c4fd946aa2 2024-11-08T02:32:14,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d8816685a1fe4dbc908726c4fd946aa2, entries=150, sequenceid=56, filesize=11.7 K 2024-11-08T02:32:14,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a2c847edb50c5b92217fdd99390ba5a5 in 1673ms, sequenceid=56, compaction requested=true 2024-11-08T02:32:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:14,436 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:14,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:14,436 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:14,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:14,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:14,437 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:14,437 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:14,437 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/B is initiating minor compaction (all files) 2024-11-08T02:32:14,438 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/A is initiating minor compaction (all files) 2024-11-08T02:32:14,438 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/A in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,438 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/B in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,438 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32b112df40954f27969d6a09d4251137, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/324a631ddfc5463ca382393065ead094, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/6db780d16d3446509ec5b3b5a6936575] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.2 K 2024-11-08T02:32:14,438 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ba3c0ffef91f4ba281f1898d4368c35c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/79a9db9f80d14bbab6d1704765f5aef2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c377d117ffb4429182fa0c914acd5157] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=99.1 K 2024-11-08T02:32:14,438 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,438 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ba3c0ffef91f4ba281f1898d4368c35c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/79a9db9f80d14bbab6d1704765f5aef2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c377d117ffb4429182fa0c914acd5157] 2024-11-08T02:32:14,438 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 32b112df40954f27969d6a09d4251137, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731033129490 2024-11-08T02:32:14,438 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ba3c0ffef91f4ba281f1898d4368c35c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731033129490 2024-11-08T02:32:14,438 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 324a631ddfc5463ca382393065ead094, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033129506 2024-11-08T02:32:14,438 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 79a9db9f80d14bbab6d1704765f5aef2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033129506 2024-11-08T02:32:14,439 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6db780d16d3446509ec5b3b5a6936575, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731033131642 2024-11-08T02:32:14,439 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting c377d117ffb4429182fa0c914acd5157, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731033131641 2024-11-08T02:32:14,445 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:14,446 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:14,447 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/be9d6779207d47478b99c3a054cc89a0 is 50, key is test_row_0/B:col10/1731033132762/Put/seqid=0 2024-11-08T02:32:14,449 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108e8350667df6b4493a17e47b50f92a1b1_a2c847edb50c5b92217fdd99390ba5a5 store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:14,451 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108e8350667df6b4493a17e47b50f92a1b1_a2c847edb50c5b92217fdd99390ba5a5, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:14,452 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e8350667df6b4493a17e47b50f92a1b1_a2c847edb50c5b92217fdd99390ba5a5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:14,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742130_1306 (size=12104) 2024-11-08T02:32:14,462 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/be9d6779207d47478b99c3a054cc89a0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/be9d6779207d47478b99c3a054cc89a0 2024-11-08T02:32:14,469 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/B of a2c847edb50c5b92217fdd99390ba5a5 into be9d6779207d47478b99c3a054cc89a0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:14,469 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:14,469 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/B, priority=13, startTime=1731033134436; duration=0sec 2024-11-08T02:32:14,469 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:14,469 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:B 2024-11-08T02:32:14,469 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:14,471 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:14,471 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/C is initiating minor compaction (all files) 2024-11-08T02:32:14,471 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/C in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,471 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a6136c2e77534c25987f0ef92e152004, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a35546371bc04e3c991961023616d879, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d8816685a1fe4dbc908726c4fd946aa2] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.2 K 2024-11-08T02:32:14,472 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a6136c2e77534c25987f0ef92e152004, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731033129490 2024-11-08T02:32:14,472 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a35546371bc04e3c991961023616d879, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033129506 2024-11-08T02:32:14,472 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting d8816685a1fe4dbc908726c4fd946aa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731033131642 2024-11-08T02:32:14,482 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:14,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742131_1307 (size=4469) 2024-11-08T02:32:14,485 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/6de814f38d94412996baf2525b6b99e0 is 50, key is test_row_0/C:col10/1731033132762/Put/seqid=0 2024-11-08T02:32:14,486 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#A#compaction#259 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:14,486 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/ed8f2903fe66411ebe993d289b09fdbb is 175, key is test_row_0/A:col10/1731033132762/Put/seqid=0 2024-11-08T02:32:14,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742132_1308 (size=12104) 2024-11-08T02:32:14,506 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-08T02:32:14,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:14,507 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:32:14,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:14,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742133_1309 (size=31058) 2024-11-08T02:32:14,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:14,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:14,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:14,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:14,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:14,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110882818d9bdc49469681f0bc8bb620a6b2_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033132795/Put/seqid=0 2024-11-08T02:32:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742134_1310 (size=12154) 2024-11-08T02:32:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-08T02:32:14,896 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/6de814f38d94412996baf2525b6b99e0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/6de814f38d94412996baf2525b6b99e0 2024-11-08T02:32:14,900 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/C of a2c847edb50c5b92217fdd99390ba5a5 into 6de814f38d94412996baf2525b6b99e0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:14,900 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:14,900 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/C, priority=13, startTime=1731033134436; duration=0sec 2024-11-08T02:32:14,900 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:14,900 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:C 2024-11-08T02:32:14,911 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/ed8f2903fe66411ebe993d289b09fdbb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ed8f2903fe66411ebe993d289b09fdbb 2024-11-08T02:32:14,915 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/A of a2c847edb50c5b92217fdd99390ba5a5 into ed8f2903fe66411ebe993d289b09fdbb(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:14,915 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:14,915 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/A, priority=13, startTime=1731033134436; duration=0sec 2024-11-08T02:32:14,915 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:14,915 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:A 2024-11-08T02:32:14,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:14,923 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110882818d9bdc49469681f0bc8bb620a6b2_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882818d9bdc49469681f0bc8bb620a6b2_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:14,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/d022c0b9320e4379a015b712046116f0, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:14,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/d022c0b9320e4379a015b712046116f0 is 175, key is test_row_0/A:col10/1731033132795/Put/seqid=0 2024-11-08T02:32:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:14,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:14,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742135_1311 (size=30955) 2024-11-08T02:32:14,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033194937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033194941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033194942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:14,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033194943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033195044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033195045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033195048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033195048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033195249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033195250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033195251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033195251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,340 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/d022c0b9320e4379a015b712046116f0 2024-11-08T02:32:15,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/7f187c8a215c40dfa3496ccf398470f2 is 50, key is test_row_0/B:col10/1731033132795/Put/seqid=0 2024-11-08T02:32:15,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742136_1312 (size=12001) 2024-11-08T02:32:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033195556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033195556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033195556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033195556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-08T02:32:15,749 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/7f187c8a215c40dfa3496ccf398470f2 2024-11-08T02:32:15,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/21606be01c294715a55f455f03793b06 is 50, key is test_row_0/C:col10/1731033132795/Put/seqid=0 2024-11-08T02:32:15,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742137_1313 (size=12001) 2024-11-08T02:32:16,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:16,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033196060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:16,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:16,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033196061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033196064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:16,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:16,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033196064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:16,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/21606be01c294715a55f455f03793b06 2024-11-08T02:32:16,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/d022c0b9320e4379a015b712046116f0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/d022c0b9320e4379a015b712046116f0 2024-11-08T02:32:16,166 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/d022c0b9320e4379a015b712046116f0, entries=150, sequenceid=77, filesize=30.2 K 2024-11-08T02:32:16,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/7f187c8a215c40dfa3496ccf398470f2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/7f187c8a215c40dfa3496ccf398470f2 2024-11-08T02:32:16,170 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/7f187c8a215c40dfa3496ccf398470f2, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T02:32:16,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/21606be01c294715a55f455f03793b06 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/21606be01c294715a55f455f03793b06 2024-11-08T02:32:16,174 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/21606be01c294715a55f455f03793b06, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T02:32:16,174 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a2c847edb50c5b92217fdd99390ba5a5 in 1667ms, sequenceid=77, compaction requested=false 2024-11-08T02:32:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:16,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:16,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=103 2024-11-08T02:32:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=103 2024-11-08T02:32:16,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=103, resume processing ppid=102 2024-11-08T02:32:16,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=103, ppid=102, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5820 sec 2024-11-08T02:32:16,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=102, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees in 2.5860 sec 2024-11-08T02:32:16,298 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T02:32:16,298 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T02:32:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:17,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:32:17,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:17,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:17,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:17,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:17,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:17,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:17,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110895012848b0d64aedbc6af7367bace4b4_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033137069/Put/seqid=0 2024-11-08T02:32:17,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742138_1314 (size=14594) 2024-11-08T02:32:17,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033197093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033197094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033197095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033197095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033197198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033197199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033197200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033197202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033197404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033197404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033197404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033197404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:17,484 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110895012848b0d64aedbc6af7367bace4b4_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110895012848b0d64aedbc6af7367bace4b4_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:17,485 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/4632f3df948a437792d8e54eaf7092ff, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:17,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/4632f3df948a437792d8e54eaf7092ff is 175, key is test_row_0/A:col10/1731033137069/Put/seqid=0 2024-11-08T02:32:17,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742139_1315 (size=39549) 2024-11-08T02:32:17,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033197687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,690 DEBUG [Thread-1356 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8182 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-08T02:32:17,696 INFO [Thread-1364 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 102 completed 2024-11-08T02:32:17,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees 2024-11-08T02:32:17,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-08T02:32:17,698 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=104, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:17,699 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=104, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:17,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:17,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033197708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033197709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033197710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:17,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033197710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-08T02:32:17,850 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:17,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-08T02:32:17,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:17,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:17,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:17,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:17,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:17,893 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/4632f3df948a437792d8e54eaf7092ff 2024-11-08T02:32:17,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/b23f86a4035944e69e95052b98684672 is 50, key is test_row_0/B:col10/1731033137069/Put/seqid=0 2024-11-08T02:32:17,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742140_1316 (size=12001) 2024-11-08T02:32:18,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-08T02:32:18,002 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-08T02:32:18,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:18,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,154 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-08T02:32:18,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:18,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033198213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033198214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:18,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033198214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:18,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033198215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-08T02:32:18,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/b23f86a4035944e69e95052b98684672 2024-11-08T02:32:18,306 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-08T02:32:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/32be975829ee4fde8e64354a335dfadd is 50, key is test_row_0/C:col10/1731033137069/Put/seqid=0 2024-11-08T02:32:18,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742141_1317 (size=12001) 2024-11-08T02:32:18,461 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-08T02:32:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,613 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-08T02:32:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:18,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:18,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/32be975829ee4fde8e64354a335dfadd 2024-11-08T02:32:18,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/4632f3df948a437792d8e54eaf7092ff as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/4632f3df948a437792d8e54eaf7092ff 2024-11-08T02:32:18,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/4632f3df948a437792d8e54eaf7092ff, entries=200, sequenceid=96, filesize=38.6 K 2024-11-08T02:32:18,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/b23f86a4035944e69e95052b98684672 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b23f86a4035944e69e95052b98684672 2024-11-08T02:32:18,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b23f86a4035944e69e95052b98684672, entries=150, sequenceid=96, filesize=11.7 K 2024-11-08T02:32:18,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/32be975829ee4fde8e64354a335dfadd as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/32be975829ee4fde8e64354a335dfadd 2024-11-08T02:32:18,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/32be975829ee4fde8e64354a335dfadd, entries=150, sequenceid=96, filesize=11.7 K 2024-11-08T02:32:18,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a2c847edb50c5b92217fdd99390ba5a5 in 1658ms, sequenceid=96, compaction requested=true 2024-11-08T02:32:18,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:18,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:18,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:18,728 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:18,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:18,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:18,728 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:18,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:18,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:18,730 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:18,730 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:18,730 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/A is initiating minor compaction (all files) 2024-11-08T02:32:18,730 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/B is initiating minor compaction (all files) 2024-11-08T02:32:18,730 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/B in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,730 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/A in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,730 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ed8f2903fe66411ebe993d289b09fdbb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/d022c0b9320e4379a015b712046116f0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/4632f3df948a437792d8e54eaf7092ff] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=99.2 K 2024-11-08T02:32:18,730 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/be9d6779207d47478b99c3a054cc89a0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/7f187c8a215c40dfa3496ccf398470f2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b23f86a4035944e69e95052b98684672] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.3 K 2024-11-08T02:32:18,730 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,730 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ed8f2903fe66411ebe993d289b09fdbb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/d022c0b9320e4379a015b712046116f0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/4632f3df948a437792d8e54eaf7092ff] 2024-11-08T02:32:18,730 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting be9d6779207d47478b99c3a054cc89a0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731033131642 2024-11-08T02:32:18,731 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ed8f2903fe66411ebe993d289b09fdbb, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731033131642 2024-11-08T02:32:18,731 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 7f187c8a215c40dfa3496ccf398470f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731033132769 2024-11-08T02:32:18,731 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d022c0b9320e4379a015b712046116f0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731033132769 2024-11-08T02:32:18,731 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b23f86a4035944e69e95052b98684672, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731033134935 2024-11-08T02:32:18,731 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4632f3df948a437792d8e54eaf7092ff, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731033134935 2024-11-08T02:32:18,741 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:18,741 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/52fb3a17d5fd4fd5b394226cc90e8ed7 is 50, key is test_row_0/B:col10/1731033137069/Put/seqid=0 2024-11-08T02:32:18,751 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742142_1318 (size=12207) 2024-11-08T02:32:18,763 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108ed155f9a58d144fbb6056dac930d39e4_a2c847edb50c5b92217fdd99390ba5a5 store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:18,765 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:18,765 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108ed155f9a58d144fbb6056dac930d39e4_a2c847edb50c5b92217fdd99390ba5a5, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:18,765 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ed155f9a58d144fbb6056dac930d39e4_a2c847edb50c5b92217fdd99390ba5a5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:18,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-08T02:32:18,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,766 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T02:32:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:18,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:18,774 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/52fb3a17d5fd4fd5b394226cc90e8ed7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/52fb3a17d5fd4fd5b394226cc90e8ed7 2024-11-08T02:32:18,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c5750ff5ccd445c3be27f1bbd846ee8e_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033137094/Put/seqid=0 2024-11-08T02:32:18,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742143_1319 (size=4469) 2024-11-08T02:32:18,787 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/B of a2c847edb50c5b92217fdd99390ba5a5 into 52fb3a17d5fd4fd5b394226cc90e8ed7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:18,787 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:18,787 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/B, priority=13, startTime=1731033138728; duration=0sec 2024-11-08T02:32:18,787 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:18,787 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:B 2024-11-08T02:32:18,787 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:18,789 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:18,789 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/C is initiating minor compaction (all files) 2024-11-08T02:32:18,789 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/C in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:18,789 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/6de814f38d94412996baf2525b6b99e0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/21606be01c294715a55f455f03793b06, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/32be975829ee4fde8e64354a335dfadd] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.3 K 2024-11-08T02:32:18,789 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6de814f38d94412996baf2525b6b99e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731033131642 2024-11-08T02:32:18,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742144_1320 (size=12154) 2024-11-08T02:32:18,792 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 21606be01c294715a55f455f03793b06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731033132769 2024-11-08T02:32:18,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:18,793 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 32be975829ee4fde8e64354a335dfadd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731033134935 2024-11-08T02:32:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-08T02:32:18,802 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c5750ff5ccd445c3be27f1bbd846ee8e_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c5750ff5ccd445c3be27f1bbd846ee8e_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:18,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/14e0c3d0990949b9ae85250efe32af9f, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:18,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/14e0c3d0990949b9ae85250efe32af9f is 175, key is test_row_0/A:col10/1731033137094/Put/seqid=0 2024-11-08T02:32:18,806 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:18,806 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/cf02f29df538457c87094501be443b81 is 50, key is test_row_0/C:col10/1731033137069/Put/seqid=0 2024-11-08T02:32:18,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742145_1321 (size=30955) 2024-11-08T02:32:18,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742146_1322 (size=12207) 2024-11-08T02:32:19,182 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#A#compaction#268 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:19,183 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/afb680f0a11d463c86883df96db7d970 is 175, key is test_row_0/A:col10/1731033137069/Put/seqid=0 2024-11-08T02:32:19,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742147_1323 (size=31161) 2024-11-08T02:32:19,191 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/afb680f0a11d463c86883df96db7d970 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/afb680f0a11d463c86883df96db7d970 2024-11-08T02:32:19,195 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/A of a2c847edb50c5b92217fdd99390ba5a5 into afb680f0a11d463c86883df96db7d970(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:19,195 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:19,195 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/A, priority=13, startTime=1731033138728; duration=0sec 2024-11-08T02:32:19,195 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:19,195 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:A 2024-11-08T02:32:19,207 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/14e0c3d0990949b9ae85250efe32af9f 2024-11-08T02:32:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/413b39177c9943e1b8528c688e4418d5 is 50, key is test_row_0/B:col10/1731033137094/Put/seqid=0 2024-11-08T02:32:19,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742148_1324 (size=12001) 2024-11-08T02:32:19,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:19,222 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/cf02f29df538457c87094501be443b81 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/cf02f29df538457c87094501be443b81 2024-11-08T02:32:19,227 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/C of a2c847edb50c5b92217fdd99390ba5a5 into cf02f29df538457c87094501be443b81(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:19,227 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:19,227 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/C, priority=13, startTime=1731033138728; duration=0sec 2024-11-08T02:32:19,227 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:19,227 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:C 2024-11-08T02:32:19,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033199238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033199238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033199242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033199242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033199344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033199344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033199344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033199348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033199547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033199547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033199548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033199553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,621 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/413b39177c9943e1b8528c688e4418d5 2024-11-08T02:32:19,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/5299437763cd42f293939bd034ba89b2 is 50, key is test_row_0/C:col10/1731033137094/Put/seqid=0 2024-11-08T02:32:19,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742149_1325 (size=12001) 2024-11-08T02:32:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-08T02:32:19,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033199851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033199852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033199852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033199857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,032 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/5299437763cd42f293939bd034ba89b2 2024-11-08T02:32:20,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/14e0c3d0990949b9ae85250efe32af9f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/14e0c3d0990949b9ae85250efe32af9f 2024-11-08T02:32:20,040 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/14e0c3d0990949b9ae85250efe32af9f, entries=150, sequenceid=115, filesize=30.2 K 2024-11-08T02:32:20,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/413b39177c9943e1b8528c688e4418d5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/413b39177c9943e1b8528c688e4418d5 2024-11-08T02:32:20,044 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/413b39177c9943e1b8528c688e4418d5, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:32:20,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/5299437763cd42f293939bd034ba89b2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/5299437763cd42f293939bd034ba89b2 2024-11-08T02:32:20,048 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/5299437763cd42f293939bd034ba89b2, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:32:20,048 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a2c847edb50c5b92217fdd99390ba5a5 in 1282ms, sequenceid=115, compaction requested=false 2024-11-08T02:32:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105 2024-11-08T02:32:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=105 2024-11-08T02:32:20,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=105, resume processing ppid=104 2024-11-08T02:32:20,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=105, ppid=104, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3500 sec 2024-11-08T02:32:20,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=104, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees in 2.3540 sec 2024-11-08T02:32:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:20,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:32:20,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:20,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:20,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:20,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:20,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:20,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:20,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084e35c434704d403db905497d03130b82_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033140358/Put/seqid=0 2024-11-08T02:32:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742150_1326 (size=17284) 2024-11-08T02:32:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033200374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033200378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033200379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033200380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033200481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033200481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033200485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033200486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033200683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033200684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033200693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033200694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,768 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:20,771 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084e35c434704d403db905497d03130b82_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084e35c434704d403db905497d03130b82_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:20,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/b63f6d9e570743bdb98637e96e16d390, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:20,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/b63f6d9e570743bdb98637e96e16d390 is 175, key is test_row_0/A:col10/1731033140358/Put/seqid=0 2024-11-08T02:32:20,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742151_1327 (size=48389) 2024-11-08T02:32:20,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033200992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:20,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033200993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033201002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:21,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033201002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,184 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/b63f6d9e570743bdb98637e96e16d390 2024-11-08T02:32:21,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/feebcb49533142ab80e7bd88c6ccb9bd is 50, key is test_row_0/B:col10/1731033140358/Put/seqid=0 2024-11-08T02:32:21,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742152_1328 (size=12151) 2024-11-08T02:32:21,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033201495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033201503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033201507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033201509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/feebcb49533142ab80e7bd88c6ccb9bd 2024-11-08T02:32:21,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/358b8359fa2644408209a1c3330ed938 is 50, key is test_row_0/C:col10/1731033140358/Put/seqid=0 2024-11-08T02:32:21,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742153_1329 (size=12151) 2024-11-08T02:32:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-08T02:32:21,802 INFO [Thread-1364 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 104 completed 2024-11-08T02:32:21,803 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees 2024-11-08T02:32:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-08T02:32:21,804 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=106, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:21,805 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=106, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:21,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:21,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-08T02:32:21,956 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:21,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-08T02:32:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:21,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:21,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:22,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/358b8359fa2644408209a1c3330ed938 2024-11-08T02:32:22,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/b63f6d9e570743bdb98637e96e16d390 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/b63f6d9e570743bdb98637e96e16d390 2024-11-08T02:32:22,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/b63f6d9e570743bdb98637e96e16d390, entries=250, sequenceid=137, filesize=47.3 K 2024-11-08T02:32:22,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/feebcb49533142ab80e7bd88c6ccb9bd as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/feebcb49533142ab80e7bd88c6ccb9bd 2024-11-08T02:32:22,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/feebcb49533142ab80e7bd88c6ccb9bd, entries=150, sequenceid=137, filesize=11.9 K 2024-11-08T02:32:22,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/358b8359fa2644408209a1c3330ed938 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/358b8359fa2644408209a1c3330ed938 2024-11-08T02:32:22,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/358b8359fa2644408209a1c3330ed938, entries=150, sequenceid=137, filesize=11.9 K 2024-11-08T02:32:22,019 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a2c847edb50c5b92217fdd99390ba5a5 in 1661ms, sequenceid=137, compaction requested=true 2024-11-08T02:32:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:22,019 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:22,019 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:22,020 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:22,020 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/B is initiating minor compaction (all files) 2024-11-08T02:32:22,020 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/B in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:22,020 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/52fb3a17d5fd4fd5b394226cc90e8ed7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/413b39177c9943e1b8528c688e4418d5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/feebcb49533142ab80e7bd88c6ccb9bd] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.5 K 2024-11-08T02:32:22,020 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:22,020 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/A is initiating minor compaction (all files) 2024-11-08T02:32:22,020 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/A in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:22,020 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/afb680f0a11d463c86883df96db7d970, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/14e0c3d0990949b9ae85250efe32af9f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/b63f6d9e570743bdb98637e96e16d390] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=107.9 K 2024-11-08T02:32:22,020 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:22,020 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/afb680f0a11d463c86883df96db7d970, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/14e0c3d0990949b9ae85250efe32af9f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/b63f6d9e570743bdb98637e96e16d390] 2024-11-08T02:32:22,020 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 52fb3a17d5fd4fd5b394226cc90e8ed7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731033134935 2024-11-08T02:32:22,020 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting afb680f0a11d463c86883df96db7d970, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731033134935 2024-11-08T02:32:22,021 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 413b39177c9943e1b8528c688e4418d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033137086 2024-11-08T02:32:22,021 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 14e0c3d0990949b9ae85250efe32af9f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033137086 2024-11-08T02:32:22,021 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting feebcb49533142ab80e7bd88c6ccb9bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033139236 2024-11-08T02:32:22,021 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b63f6d9e570743bdb98637e96e16d390, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033139236 2024-11-08T02:32:22,026 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:22,026 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/1d1f78a3e3924af1a7f9b5a485e47d36 is 50, key is test_row_0/B:col10/1731033140358/Put/seqid=0 2024-11-08T02:32:22,027 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:22,030 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108675333f9d0fe4bd096ad22f576ea9796_a2c847edb50c5b92217fdd99390ba5a5 store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:22,032 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108675333f9d0fe4bd096ad22f576ea9796_a2c847edb50c5b92217fdd99390ba5a5, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:22,032 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108675333f9d0fe4bd096ad22f576ea9796_a2c847edb50c5b92217fdd99390ba5a5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:22,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742154_1330 (size=12459) 2024-11-08T02:32:22,037 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/1d1f78a3e3924af1a7f9b5a485e47d36 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/1d1f78a3e3924af1a7f9b5a485e47d36 2024-11-08T02:32:22,040 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/B of a2c847edb50c5b92217fdd99390ba5a5 into 1d1f78a3e3924af1a7f9b5a485e47d36(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:22,041 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:22,041 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/B, priority=13, startTime=1731033142019; duration=0sec 2024-11-08T02:32:22,041 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:22,041 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:B 2024-11-08T02:32:22,041 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:22,041 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:22,041 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/C is initiating minor compaction (all files) 2024-11-08T02:32:22,042 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/C in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:22,042 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/cf02f29df538457c87094501be443b81, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/5299437763cd42f293939bd034ba89b2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/358b8359fa2644408209a1c3330ed938] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.5 K 2024-11-08T02:32:22,042 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting cf02f29df538457c87094501be443b81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731033134935 2024-11-08T02:32:22,043 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 5299437763cd42f293939bd034ba89b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033137086 2024-11-08T02:32:22,043 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 358b8359fa2644408209a1c3330ed938, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033139236 2024-11-08T02:32:22,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742155_1331 (size=4469) 2024-11-08T02:32:22,049 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:22,050 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/0bcd0afde7b54885b3910886e659c7e3 is 50, key is test_row_0/C:col10/1731033140358/Put/seqid=0 2024-11-08T02:32:22,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742156_1332 (size=12459) 2024-11-08T02:32:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-08T02:32:22,108 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-08T02:32:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:22,109 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:32:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:22,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108421cdd41f9294f96a8bec1928b63e814_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033140379/Put/seqid=0 2024-11-08T02:32:22,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742157_1333 (size=12304) 2024-11-08T02:32:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-08T02:32:22,443 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#A#compaction#277 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:22,444 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/8a2a85c7b5c047fd9ae9a72ef07c8400 is 175, key is test_row_0/A:col10/1731033140358/Put/seqid=0 2024-11-08T02:32:22,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742158_1334 (size=31413) 2024-11-08T02:32:22,467 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/0bcd0afde7b54885b3910886e659c7e3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/0bcd0afde7b54885b3910886e659c7e3 2024-11-08T02:32:22,471 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/C of a2c847edb50c5b92217fdd99390ba5a5 into 0bcd0afde7b54885b3910886e659c7e3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:22,471 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:22,471 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/C, priority=13, startTime=1731033142019; duration=0sec 2024-11-08T02:32:22,471 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:22,471 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:C 2024-11-08T02:32:22,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:22,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:22,528 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108421cdd41f9294f96a8bec1928b63e814_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108421cdd41f9294f96a8bec1928b63e814_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/cd97bbc184924ed99d03508b050693dc, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:22,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/cd97bbc184924ed99d03508b050693dc is 175, key is test_row_0/A:col10/1731033140379/Put/seqid=0 2024-11-08T02:32:22,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742159_1335 (size=31105) 2024-11-08T02:32:22,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033202533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033202534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033202535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033202541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033202642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033202642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033202643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033202644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033202844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033202844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033202845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,851 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/8a2a85c7b5c047fd9ae9a72ef07c8400 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/8a2a85c7b5c047fd9ae9a72ef07c8400 2024-11-08T02:32:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:22,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033202850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:22,855 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/A of a2c847edb50c5b92217fdd99390ba5a5 into 8a2a85c7b5c047fd9ae9a72ef07c8400(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:22,855 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:22,855 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/A, priority=13, startTime=1731033142019; duration=0sec 2024-11-08T02:32:22,855 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:22,855 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:A 2024-11-08T02:32:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-08T02:32:22,934 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/cd97bbc184924ed99d03508b050693dc 2024-11-08T02:32:22,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/37d29fa6735c41e0ae66cc1e9c18c5dc is 50, key is test_row_0/B:col10/1731033140379/Put/seqid=0 2024-11-08T02:32:22,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742160_1336 (size=12151) 2024-11-08T02:32:22,945 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/37d29fa6735c41e0ae66cc1e9c18c5dc 2024-11-08T02:32:22,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/81e34ad68f3742769aad85574967527f is 50, key is test_row_0/C:col10/1731033140379/Put/seqid=0 2024-11-08T02:32:22,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742161_1337 (size=12151) 2024-11-08T02:32:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033203147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033203147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033203152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033203154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,362 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/81e34ad68f3742769aad85574967527f 2024-11-08T02:32:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/cd97bbc184924ed99d03508b050693dc as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/cd97bbc184924ed99d03508b050693dc 2024-11-08T02:32:23,369 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/cd97bbc184924ed99d03508b050693dc, entries=150, sequenceid=155, filesize=30.4 K 2024-11-08T02:32:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/37d29fa6735c41e0ae66cc1e9c18c5dc as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/37d29fa6735c41e0ae66cc1e9c18c5dc 2024-11-08T02:32:23,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/37d29fa6735c41e0ae66cc1e9c18c5dc, entries=150, sequenceid=155, filesize=11.9 K 2024-11-08T02:32:23,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/81e34ad68f3742769aad85574967527f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/81e34ad68f3742769aad85574967527f 2024-11-08T02:32:23,376 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/81e34ad68f3742769aad85574967527f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-08T02:32:23,377 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a2c847edb50c5b92217fdd99390ba5a5 in 1268ms, sequenceid=155, compaction requested=false 2024-11-08T02:32:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:23,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-11-08T02:32:23,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-11-08T02:32:23,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-11-08T02:32:23,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5730 sec 2024-11-08T02:32:23,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees in 1.5760 sec 2024-11-08T02:32:23,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:23,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:32:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:23,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:23,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110845301f5821334ed4b280cab8743c27c2_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033142527/Put/seqid=0 2024-11-08T02:32:23,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742162_1338 (size=17284) 2024-11-08T02:32:23,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033203669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033203670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033203671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033203676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033203777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033203777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033203782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033203782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-08T02:32:23,908 INFO [Thread-1364 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 106 completed 2024-11-08T02:32:23,909 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees 2024-11-08T02:32:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-08T02:32:23,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=108, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:23,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=108, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:23,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033203982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033203981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033203988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:23,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033203988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-08T02:32:24,061 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-08T02:32:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,066 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:24,069 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110845301f5821334ed4b280cab8743c27c2_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845301f5821334ed4b280cab8743c27c2_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:24,069 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/6f1ba1afe5584ade85bf98bebb2cea62, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:24,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/6f1ba1afe5584ade85bf98bebb2cea62 is 175, key is test_row_0/A:col10/1731033142527/Put/seqid=0 2024-11-08T02:32:24,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742163_1339 (size=48389) 2024-11-08T02:32:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-08T02:32:24,213 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-08T02:32:24,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:24,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033204291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033204291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033204295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033204295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,366 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-08T02:32:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/6f1ba1afe5584ade85bf98bebb2cea62 2024-11-08T02:32:24,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/bc21583eae1d4218b9b24170b62ab04e is 50, key is test_row_0/B:col10/1731033142527/Put/seqid=0 2024-11-08T02:32:24,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742164_1340 (size=12151) 2024-11-08T02:32:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-08T02:32:24,518 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-08T02:32:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:24,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,670 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-08T02:32:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033204796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033204798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033204802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033204804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,823 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-08T02:32:24,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:24,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:24,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/bc21583eae1d4218b9b24170b62ab04e 2024-11-08T02:32:24,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/9632f026082f4d41a7b4c419f4a8d55f is 50, key is test_row_0/C:col10/1731033142527/Put/seqid=0 2024-11-08T02:32:24,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742165_1341 (size=12151) 2024-11-08T02:32:24,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/9632f026082f4d41a7b4c419f4a8d55f 2024-11-08T02:32:24,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/6f1ba1afe5584ade85bf98bebb2cea62 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/6f1ba1afe5584ade85bf98bebb2cea62 2024-11-08T02:32:24,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/6f1ba1afe5584ade85bf98bebb2cea62, entries=250, sequenceid=178, filesize=47.3 K 2024-11-08T02:32:24,900 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1000): StoreScanner already closing. There is no need to updateReaders 2024-11-08T02:32:24,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/bc21583eae1d4218b9b24170b62ab04e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/bc21583eae1d4218b9b24170b62ab04e 2024-11-08T02:32:24,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/bc21583eae1d4218b9b24170b62ab04e, entries=150, sequenceid=178, filesize=11.9 K 2024-11-08T02:32:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/9632f026082f4d41a7b4c419f4a8d55f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/9632f026082f4d41a7b4c419f4a8d55f 2024-11-08T02:32:24,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/9632f026082f4d41a7b4c419f4a8d55f, entries=150, sequenceid=178, filesize=11.9 K 2024-11-08T02:32:24,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for a2c847edb50c5b92217fdd99390ba5a5 in 1253ms, sequenceid=178, compaction requested=true 2024-11-08T02:32:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:24,909 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:24,909 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:24,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110907 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/A is initiating minor compaction (all files) 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/B is initiating minor compaction (all files) 2024-11-08T02:32:24,910 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/B in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,910 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/A in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,910 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/8a2a85c7b5c047fd9ae9a72ef07c8400, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/cd97bbc184924ed99d03508b050693dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/6f1ba1afe5584ade85bf98bebb2cea62] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=108.3 K 2024-11-08T02:32:24,910 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/1d1f78a3e3924af1a7f9b5a485e47d36, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/37d29fa6735c41e0ae66cc1e9c18c5dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/bc21583eae1d4218b9b24170b62ab04e] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.9 K 2024-11-08T02:32:24,910 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/8a2a85c7b5c047fd9ae9a72ef07c8400, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/cd97bbc184924ed99d03508b050693dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/6f1ba1afe5584ade85bf98bebb2cea62] 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1d1f78a3e3924af1a7f9b5a485e47d36, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033139236 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8a2a85c7b5c047fd9ae9a72ef07c8400, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033139236 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 37d29fa6735c41e0ae66cc1e9c18c5dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731033140373 2024-11-08T02:32:24,910 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting cd97bbc184924ed99d03508b050693dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731033140373 2024-11-08T02:32:24,911 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting bc21583eae1d4218b9b24170b62ab04e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731033142527 2024-11-08T02:32:24,911 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6f1ba1afe5584ade85bf98bebb2cea62, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731033142527 2024-11-08T02:32:24,918 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:24,918 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/f758a0c1c737420eb2f474d94f253c1a is 50, key is test_row_0/B:col10/1731033142527/Put/seqid=0 2024-11-08T02:32:24,919 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:24,929 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411089021026472894a83abf8c272757fa4fe_a2c847edb50c5b92217fdd99390ba5a5 store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:24,931 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411089021026472894a83abf8c272757fa4fe_a2c847edb50c5b92217fdd99390ba5a5, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:24,931 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411089021026472894a83abf8c272757fa4fe_a2c847edb50c5b92217fdd99390ba5a5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:24,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742166_1342 (size=12561) 2024-11-08T02:32:24,937 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/f758a0c1c737420eb2f474d94f253c1a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/f758a0c1c737420eb2f474d94f253c1a 2024-11-08T02:32:24,942 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/B of a2c847edb50c5b92217fdd99390ba5a5 into f758a0c1c737420eb2f474d94f253c1a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:24,942 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:24,942 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/B, priority=13, startTime=1731033144909; duration=0sec 2024-11-08T02:32:24,942 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:24,942 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:B 2024-11-08T02:32:24,942 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:24,944 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:24,945 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/C is initiating minor compaction (all files) 2024-11-08T02:32:24,945 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/C in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,945 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/0bcd0afde7b54885b3910886e659c7e3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/81e34ad68f3742769aad85574967527f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/9632f026082f4d41a7b4c419f4a8d55f] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=35.9 K 2024-11-08T02:32:24,945 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 0bcd0afde7b54885b3910886e659c7e3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731033139236 2024-11-08T02:32:24,945 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 81e34ad68f3742769aad85574967527f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731033140373 2024-11-08T02:32:24,946 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 9632f026082f4d41a7b4c419f4a8d55f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731033142527 2024-11-08T02:32:24,952 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#C#compaction#287 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:24,953 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/83b301020b894c859371064969d04150 is 50, key is test_row_0/C:col10/1731033142527/Put/seqid=0 2024-11-08T02:32:24,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742167_1343 (size=4469) 2024-11-08T02:32:24,957 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#A#compaction#286 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:24,957 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/c855dcb9d7614582872625dcfee0d7a2 is 175, key is test_row_0/A:col10/1731033142527/Put/seqid=0 2024-11-08T02:32:24,975 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:24,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-08T02:32:24,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:24,975 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:32:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:24,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742168_1344 (size=12561) 2024-11-08T02:32:24,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742169_1345 (size=31515) 2024-11-08T02:32:24,985 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/83b301020b894c859371064969d04150 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/83b301020b894c859371064969d04150 2024-11-08T02:32:24,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d034f0189aa14a89b89d18b233961058_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033143668/Put/seqid=0 2024-11-08T02:32:24,993 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/C of a2c847edb50c5b92217fdd99390ba5a5 into 83b301020b894c859371064969d04150(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:24,993 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:24,993 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/C, priority=13, startTime=1731033144909; duration=0sec 2024-11-08T02:32:24,993 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:24,993 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:C 2024-11-08T02:32:24,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742170_1346 (size=12304) 2024-11-08T02:32:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:24,996 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/c855dcb9d7614582872625dcfee0d7a2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c855dcb9d7614582872625dcfee0d7a2 2024-11-08T02:32:24,999 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d034f0189aa14a89b89d18b233961058_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d034f0189aa14a89b89d18b233961058_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:25,001 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/A of a2c847edb50c5b92217fdd99390ba5a5 into c855dcb9d7614582872625dcfee0d7a2(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:25,001 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:25,001 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/A, priority=13, startTime=1731033144909; duration=0sec 2024-11-08T02:32:25,001 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:25,001 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:A 2024-11-08T02:32:25,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f37c630dc6524ab48ab6efedf6b7d918, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:25,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f37c630dc6524ab48ab6efedf6b7d918 is 175, key is test_row_0/A:col10/1731033143668/Put/seqid=0 2024-11-08T02:32:25,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742171_1347 (size=31105) 2024-11-08T02:32:25,007 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f37c630dc6524ab48ab6efedf6b7d918 2024-11-08T02:32:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-08T02:32:25,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/612a4485877d4bb595c4752435e1dcf9 is 50, key is test_row_0/B:col10/1731033143668/Put/seqid=0 2024-11-08T02:32:25,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742172_1348 (size=12151) 2024-11-08T02:32:25,057 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:32:25,425 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/612a4485877d4bb595c4752435e1dcf9 2024-11-08T02:32:25,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/35eb8680a42240489c10a9f841613c2c is 50, key is test_row_0/C:col10/1731033143668/Put/seqid=0 2024-11-08T02:32:25,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742173_1349 (size=12151) 2024-11-08T02:32:25,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:25,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:25,836 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/35eb8680a42240489c10a9f841613c2c 2024-11-08T02:32:25,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f37c630dc6524ab48ab6efedf6b7d918 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f37c630dc6524ab48ab6efedf6b7d918 2024-11-08T02:32:25,842 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f37c630dc6524ab48ab6efedf6b7d918, entries=150, sequenceid=194, filesize=30.4 K 2024-11-08T02:32:25,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/612a4485877d4bb595c4752435e1dcf9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/612a4485877d4bb595c4752435e1dcf9 2024-11-08T02:32:25,846 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/612a4485877d4bb595c4752435e1dcf9, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T02:32:25,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/35eb8680a42240489c10a9f841613c2c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/35eb8680a42240489c10a9f841613c2c 2024-11-08T02:32:25,850 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/35eb8680a42240489c10a9f841613c2c, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T02:32:25,859 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=87.22 KB/89310 for a2c847edb50c5b92217fdd99390ba5a5 in 884ms, sequenceid=194, compaction requested=false 2024-11-08T02:32:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:25,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-11-08T02:32:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:25,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T02:32:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-11-08T02:32:25,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:25,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:25,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:25,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-11-08T02:32:25,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9500 sec 2024-11-08T02:32:25,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees in 1.9520 sec 2024-11-08T02:32:25,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110874f1a12c895242d7b656335cac59295b_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033145823/Put/seqid=0 2024-11-08T02:32:25,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742174_1350 (size=12304) 2024-11-08T02:32:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033205884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033205885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033205885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:25,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033205887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:25,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033205988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:25,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033205992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:25,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033205992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:25,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033205993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-08T02:32:26,013 INFO [Thread-1364 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 108 completed 2024-11-08T02:32:26,014 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees 2024-11-08T02:32:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-08T02:32:26,015 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=110, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:26,016 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=110, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:26,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-08T02:32:26,167 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:26,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033206190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033206195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033206196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033206196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,272 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:26,275 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110874f1a12c895242d7b656335cac59295b_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110874f1a12c895242d7b656335cac59295b_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:26,276 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/44eb8064c05040d6b727933f1d76ae1e, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:26,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/44eb8064c05040d6b727933f1d76ae1e is 175, key is test_row_0/A:col10/1731033145823/Put/seqid=0 2024-11-08T02:32:26,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742175_1351 (size=31105) 2024-11-08T02:32:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-08T02:32:26,319 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,471 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:26,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:26,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033206498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033206498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033206499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033206504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-08T02:32:26,623 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,680 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/44eb8064c05040d6b727933f1d76ae1e 2024-11-08T02:32:26,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/dcbf6588d2ae463f82ac9dc2bf73a839 is 50, key is test_row_0/B:col10/1731033145823/Put/seqid=0 2024-11-08T02:32:26,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742176_1352 (size=12151) 2024-11-08T02:32:26,775 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,927 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:26,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:26,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:26,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:26,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033207003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033207004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033207005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033207012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,080 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/dcbf6588d2ae463f82ac9dc2bf73a839 2024-11-08T02:32:27,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a456c908df434395b7a50e319a3085a6 is 50, key is test_row_0/C:col10/1731033145823/Put/seqid=0 2024-11-08T02:32:27,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742177_1353 (size=12151) 2024-11-08T02:32:27,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-08T02:32:27,232 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:27,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:27,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,384 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:27,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a456c908df434395b7a50e319a3085a6 2024-11-08T02:32:27,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/44eb8064c05040d6b727933f1d76ae1e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/44eb8064c05040d6b727933f1d76ae1e 2024-11-08T02:32:27,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/44eb8064c05040d6b727933f1d76ae1e, entries=150, sequenceid=215, filesize=30.4 K 2024-11-08T02:32:27,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/dcbf6588d2ae463f82ac9dc2bf73a839 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/dcbf6588d2ae463f82ac9dc2bf73a839 2024-11-08T02:32:27,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/dcbf6588d2ae463f82ac9dc2bf73a839, entries=150, sequenceid=215, filesize=11.9 K 2024-11-08T02:32:27,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a456c908df434395b7a50e319a3085a6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a456c908df434395b7a50e319a3085a6 2024-11-08T02:32:27,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a456c908df434395b7a50e319a3085a6, entries=150, sequenceid=215, filesize=11.9 K 2024-11-08T02:32:27,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for a2c847edb50c5b92217fdd99390ba5a5 in 1660ms, sequenceid=215, compaction requested=true 2024-11-08T02:32:27,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:27,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:27,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:27,519 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:27,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:27,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:27,519 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:27,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:27,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:27,520 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:27,520 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:27,520 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/A is initiating minor compaction (all files) 2024-11-08T02:32:27,520 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/B is initiating minor compaction (all files) 2024-11-08T02:32:27,520 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/A in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,520 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/B in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,520 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c855dcb9d7614582872625dcfee0d7a2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f37c630dc6524ab48ab6efedf6b7d918, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/44eb8064c05040d6b727933f1d76ae1e] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=91.5 K 2024-11-08T02:32:27,520 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/f758a0c1c737420eb2f474d94f253c1a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/612a4485877d4bb595c4752435e1dcf9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/dcbf6588d2ae463f82ac9dc2bf73a839] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=36.0 K 2024-11-08T02:32:27,520 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,520 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c855dcb9d7614582872625dcfee0d7a2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f37c630dc6524ab48ab6efedf6b7d918, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/44eb8064c05040d6b727933f1d76ae1e] 2024-11-08T02:32:27,520 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting c855dcb9d7614582872625dcfee0d7a2, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731033142527 2024-11-08T02:32:27,520 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f758a0c1c737420eb2f474d94f253c1a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731033142527 2024-11-08T02:32:27,521 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f37c630dc6524ab48ab6efedf6b7d918, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731033143668 2024-11-08T02:32:27,521 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 612a4485877d4bb595c4752435e1dcf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731033143668 2024-11-08T02:32:27,521 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 44eb8064c05040d6b727933f1d76ae1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731033145823 2024-11-08T02:32:27,521 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting dcbf6588d2ae463f82ac9dc2bf73a839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731033145823 2024-11-08T02:32:27,528 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:27,528 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:27,529 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/a169c30ba42049fba0b635a3357ccb59 is 50, key is test_row_0/B:col10/1731033145823/Put/seqid=0 2024-11-08T02:32:27,530 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108c2f39cebd0da4157ab62cebfe7405951_a2c847edb50c5b92217fdd99390ba5a5 store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:27,532 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108c2f39cebd0da4157ab62cebfe7405951_a2c847edb50c5b92217fdd99390ba5a5, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:27,533 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c2f39cebd0da4157ab62cebfe7405951_a2c847edb50c5b92217fdd99390ba5a5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:27,536 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-08T02:32:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,537 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:32:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:27,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:27,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742178_1354 (size=12663) 2024-11-08T02:32:27,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742179_1355 (size=4469) 2024-11-08T02:32:27,551 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#A#compaction#295 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:27,551 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/120ed9687d974910a48f5b868cb5ce96 is 175, key is test_row_0/A:col10/1731033145823/Put/seqid=0 2024-11-08T02:32:27,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083a3c0baf56da43b39eba66203ec01ec0_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033145884/Put/seqid=0 2024-11-08T02:32:27,554 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/a169c30ba42049fba0b635a3357ccb59 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a169c30ba42049fba0b635a3357ccb59 2024-11-08T02:32:27,558 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/B of a2c847edb50c5b92217fdd99390ba5a5 into a169c30ba42049fba0b635a3357ccb59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:27,558 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:27,558 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/B, priority=13, startTime=1731033147519; duration=0sec 2024-11-08T02:32:27,558 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:27,558 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:B 2024-11-08T02:32:27,558 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:27,559 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:27,559 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/C is initiating minor compaction (all files) 2024-11-08T02:32:27,560 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/C in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:27,560 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/83b301020b894c859371064969d04150, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/35eb8680a42240489c10a9f841613c2c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a456c908df434395b7a50e319a3085a6] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=36.0 K 2024-11-08T02:32:27,560 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 83b301020b894c859371064969d04150, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1731033142527 2024-11-08T02:32:27,560 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 35eb8680a42240489c10a9f841613c2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731033143668 2024-11-08T02:32:27,560 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a456c908df434395b7a50e319a3085a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731033145823 2024-11-08T02:32:27,575 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:27,576 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/e65d3561a1024143b72caf727c0c65df is 50, key is test_row_0/C:col10/1731033145823/Put/seqid=0 2024-11-08T02:32:27,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742180_1356 (size=31617) 2024-11-08T02:32:27,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742181_1357 (size=12304) 2024-11-08T02:32:27,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:27,581 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083a3c0baf56da43b39eba66203ec01ec0_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083a3c0baf56da43b39eba66203ec01ec0_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:27,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742182_1358 (size=12663) 2024-11-08T02:32:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f8dfe51021ff4a3fb7625434588a1af9, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:27,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f8dfe51021ff4a3fb7625434588a1af9 is 175, key is test_row_0/A:col10/1731033145884/Put/seqid=0 2024-11-08T02:32:27,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742183_1359 (size=31105) 2024-11-08T02:32:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:27,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. as already flushing 2024-11-08T02:32:27,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033207778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033207880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:27,980 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/120ed9687d974910a48f5b868cb5ce96 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/120ed9687d974910a48f5b868cb5ce96 2024-11-08T02:32:27,984 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/A of a2c847edb50c5b92217fdd99390ba5a5 into 120ed9687d974910a48f5b868cb5ce96(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:27,984 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:27,984 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/A, priority=13, startTime=1731033147519; duration=0sec 2024-11-08T02:32:27,984 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:27,984 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:A 2024-11-08T02:32:27,986 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/e65d3561a1024143b72caf727c0c65df as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/e65d3561a1024143b72caf727c0c65df 2024-11-08T02:32:27,988 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f8dfe51021ff4a3fb7625434588a1af9 2024-11-08T02:32:27,990 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/C of a2c847edb50c5b92217fdd99390ba5a5 into e65d3561a1024143b72caf727c0c65df(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:27,990 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:27,990 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/C, priority=13, startTime=1731033147519; duration=0sec 2024-11-08T02:32:27,990 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:27,990 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:C 2024-11-08T02:32:28,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/32de285a41324b32b37c0fd7fcaeb933 is 50, key is test_row_0/B:col10/1731033145884/Put/seqid=0 2024-11-08T02:32:28,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742184_1360 (size=12151) 2024-11-08T02:32:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52708 deadline: 1731033208013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:28,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:28,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52680 deadline: 1731033208018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:28,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52720 deadline: 1731033208019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:28,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:28,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52696 deadline: 1731033208023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:28,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033208085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-08T02:32:28,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033208392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:28,409 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/32de285a41324b32b37c0fd7fcaeb933 2024-11-08T02:32:28,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/78e4d46c59fc4f4893bf446fb9d6adce is 50, key is test_row_0/C:col10/1731033145884/Put/seqid=0 2024-11-08T02:32:28,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742185_1361 (size=12151) 2024-11-08T02:32:28,419 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/78e4d46c59fc4f4893bf446fb9d6adce 2024-11-08T02:32:28,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/f8dfe51021ff4a3fb7625434588a1af9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f8dfe51021ff4a3fb7625434588a1af9 2024-11-08T02:32:28,431 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f8dfe51021ff4a3fb7625434588a1af9, entries=150, sequenceid=232, filesize=30.4 K 2024-11-08T02:32:28,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/32de285a41324b32b37c0fd7fcaeb933 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32de285a41324b32b37c0fd7fcaeb933 2024-11-08T02:32:28,435 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32de285a41324b32b37c0fd7fcaeb933, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T02:32:28,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/78e4d46c59fc4f4893bf446fb9d6adce as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/78e4d46c59fc4f4893bf446fb9d6adce 2024-11-08T02:32:28,438 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/78e4d46c59fc4f4893bf446fb9d6adce, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T02:32:28,439 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a2c847edb50c5b92217fdd99390ba5a5 in 902ms, sequenceid=232, compaction requested=false 2024-11-08T02:32:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:28,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=111 2024-11-08T02:32:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=111 2024-11-08T02:32:28,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=111, resume processing ppid=110 2024-11-08T02:32:28,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=111, ppid=110, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4240 sec 2024-11-08T02:32:28,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=110, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees in 2.4280 sec 2024-11-08T02:32:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:28,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:32:28,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:28,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:28,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108795c8d0a25344e1bb824d3a2a325d971_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033148899/Put/seqid=0 2024-11-08T02:32:28,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742186_1362 (size=14794) 2024-11-08T02:32:28,917 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:28,921 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108795c8d0a25344e1bb824d3a2a325d971_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108795c8d0a25344e1bb824d3a2a325d971_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:28,922 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/bac870cc4f36419588dd56e1a11cc343, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:28,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/bac870cc4f36419588dd56e1a11cc343 is 175, key is test_row_0/A:col10/1731033148899/Put/seqid=0 2024-11-08T02:32:28,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742187_1363 (size=39749) 2024-11-08T02:32:28,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/bac870cc4f36419588dd56e1a11cc343 2024-11-08T02:32:28,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/a7c98435cfd643ef8645195eff52a413 is 50, key is test_row_0/B:col10/1731033148899/Put/seqid=0 2024-11-08T02:32:28,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742188_1364 (size=12151) 2024-11-08T02:32:28,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033208985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:29,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033209090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033209295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:29,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/a7c98435cfd643ef8645195eff52a413 2024-11-08T02:32:29,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a7c3c75f940a4b3b9367abf57b2b4a9b is 50, key is test_row_0/C:col10/1731033148899/Put/seqid=0 2024-11-08T02:32:29,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742189_1365 (size=12151) 2024-11-08T02:32:29,483 DEBUG [Thread-1373 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68035c67 to 127.0.0.1:56877 2024-11-08T02:32:29,483 DEBUG [Thread-1373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:29,487 DEBUG [Thread-1367 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x683f8469 to 127.0.0.1:56877 2024-11-08T02:32:29,487 DEBUG [Thread-1367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:29,488 DEBUG [Thread-1365 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f49665c to 127.0.0.1:56877 2024-11-08T02:32:29,488 DEBUG [Thread-1365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:29,490 DEBUG [Thread-1369 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75e4d3d0 to 127.0.0.1:56877 2024-11-08T02:32:29,490 DEBUG [Thread-1369 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:29,493 DEBUG [Thread-1371 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b308f62 to 127.0.0.1:56877 2024-11-08T02:32:29,493 DEBUG [Thread-1371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:29,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52736 deadline: 1731033209601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:29,635 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ba3c0ffef91f4ba281f1898d4368c35c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/79a9db9f80d14bbab6d1704765f5aef2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c377d117ffb4429182fa0c914acd5157, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ed8f2903fe66411ebe993d289b09fdbb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/d022c0b9320e4379a015b712046116f0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/4632f3df948a437792d8e54eaf7092ff, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/afb680f0a11d463c86883df96db7d970, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/14e0c3d0990949b9ae85250efe32af9f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/b63f6d9e570743bdb98637e96e16d390, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/8a2a85c7b5c047fd9ae9a72ef07c8400, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/cd97bbc184924ed99d03508b050693dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/6f1ba1afe5584ade85bf98bebb2cea62, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c855dcb9d7614582872625dcfee0d7a2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f37c630dc6524ab48ab6efedf6b7d918, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/44eb8064c05040d6b727933f1d76ae1e] to archive 2024-11-08T02:32:29,636 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:29,638 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ba3c0ffef91f4ba281f1898d4368c35c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ba3c0ffef91f4ba281f1898d4368c35c 2024-11-08T02:32:29,639 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/79a9db9f80d14bbab6d1704765f5aef2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/79a9db9f80d14bbab6d1704765f5aef2 2024-11-08T02:32:29,640 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c377d117ffb4429182fa0c914acd5157 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c377d117ffb4429182fa0c914acd5157 2024-11-08T02:32:29,641 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ed8f2903fe66411ebe993d289b09fdbb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/ed8f2903fe66411ebe993d289b09fdbb 2024-11-08T02:32:29,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/d022c0b9320e4379a015b712046116f0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/d022c0b9320e4379a015b712046116f0 2024-11-08T02:32:29,642 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/4632f3df948a437792d8e54eaf7092ff to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/4632f3df948a437792d8e54eaf7092ff 2024-11-08T02:32:29,643 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/afb680f0a11d463c86883df96db7d970 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/afb680f0a11d463c86883df96db7d970 2024-11-08T02:32:29,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/14e0c3d0990949b9ae85250efe32af9f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/14e0c3d0990949b9ae85250efe32af9f 2024-11-08T02:32:29,645 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/b63f6d9e570743bdb98637e96e16d390 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/b63f6d9e570743bdb98637e96e16d390 2024-11-08T02:32:29,646 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/8a2a85c7b5c047fd9ae9a72ef07c8400 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/8a2a85c7b5c047fd9ae9a72ef07c8400 2024-11-08T02:32:29,647 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/cd97bbc184924ed99d03508b050693dc to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/cd97bbc184924ed99d03508b050693dc 2024-11-08T02:32:29,648 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/6f1ba1afe5584ade85bf98bebb2cea62 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/6f1ba1afe5584ade85bf98bebb2cea62 2024-11-08T02:32:29,649 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c855dcb9d7614582872625dcfee0d7a2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/c855dcb9d7614582872625dcfee0d7a2 2024-11-08T02:32:29,650 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f37c630dc6524ab48ab6efedf6b7d918 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f37c630dc6524ab48ab6efedf6b7d918 2024-11-08T02:32:29,650 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/44eb8064c05040d6b727933f1d76ae1e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/44eb8064c05040d6b727933f1d76ae1e 2024-11-08T02:32:29,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32b112df40954f27969d6a09d4251137, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/324a631ddfc5463ca382393065ead094, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/be9d6779207d47478b99c3a054cc89a0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/6db780d16d3446509ec5b3b5a6936575, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/7f187c8a215c40dfa3496ccf398470f2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/52fb3a17d5fd4fd5b394226cc90e8ed7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b23f86a4035944e69e95052b98684672, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/413b39177c9943e1b8528c688e4418d5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/1d1f78a3e3924af1a7f9b5a485e47d36, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/feebcb49533142ab80e7bd88c6ccb9bd, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/37d29fa6735c41e0ae66cc1e9c18c5dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/f758a0c1c737420eb2f474d94f253c1a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/bc21583eae1d4218b9b24170b62ab04e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/612a4485877d4bb595c4752435e1dcf9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/dcbf6588d2ae463f82ac9dc2bf73a839] to archive 2024-11-08T02:32:29,653 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:29,655 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32b112df40954f27969d6a09d4251137 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32b112df40954f27969d6a09d4251137 2024-11-08T02:32:29,656 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/324a631ddfc5463ca382393065ead094 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/324a631ddfc5463ca382393065ead094 2024-11-08T02:32:29,657 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/be9d6779207d47478b99c3a054cc89a0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/be9d6779207d47478b99c3a054cc89a0 2024-11-08T02:32:29,658 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/6db780d16d3446509ec5b3b5a6936575 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/6db780d16d3446509ec5b3b5a6936575 2024-11-08T02:32:29,659 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/7f187c8a215c40dfa3496ccf398470f2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/7f187c8a215c40dfa3496ccf398470f2 2024-11-08T02:32:29,660 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/52fb3a17d5fd4fd5b394226cc90e8ed7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/52fb3a17d5fd4fd5b394226cc90e8ed7 2024-11-08T02:32:29,661 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b23f86a4035944e69e95052b98684672 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b23f86a4035944e69e95052b98684672 2024-11-08T02:32:29,662 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/413b39177c9943e1b8528c688e4418d5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/413b39177c9943e1b8528c688e4418d5 2024-11-08T02:32:29,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/1d1f78a3e3924af1a7f9b5a485e47d36 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/1d1f78a3e3924af1a7f9b5a485e47d36 2024-11-08T02:32:29,664 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/feebcb49533142ab80e7bd88c6ccb9bd to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/feebcb49533142ab80e7bd88c6ccb9bd 2024-11-08T02:32:29,665 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/37d29fa6735c41e0ae66cc1e9c18c5dc to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/37d29fa6735c41e0ae66cc1e9c18c5dc 2024-11-08T02:32:29,666 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/f758a0c1c737420eb2f474d94f253c1a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/f758a0c1c737420eb2f474d94f253c1a 2024-11-08T02:32:29,667 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/bc21583eae1d4218b9b24170b62ab04e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/bc21583eae1d4218b9b24170b62ab04e 2024-11-08T02:32:29,668 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/612a4485877d4bb595c4752435e1dcf9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/612a4485877d4bb595c4752435e1dcf9 2024-11-08T02:32:29,669 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/dcbf6588d2ae463f82ac9dc2bf73a839 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/dcbf6588d2ae463f82ac9dc2bf73a839 2024-11-08T02:32:29,672 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a6136c2e77534c25987f0ef92e152004, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a35546371bc04e3c991961023616d879, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/6de814f38d94412996baf2525b6b99e0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d8816685a1fe4dbc908726c4fd946aa2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/21606be01c294715a55f455f03793b06, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/cf02f29df538457c87094501be443b81, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/32be975829ee4fde8e64354a335dfadd, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/5299437763cd42f293939bd034ba89b2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/0bcd0afde7b54885b3910886e659c7e3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/358b8359fa2644408209a1c3330ed938, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/81e34ad68f3742769aad85574967527f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/83b301020b894c859371064969d04150, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/9632f026082f4d41a7b4c419f4a8d55f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/35eb8680a42240489c10a9f841613c2c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a456c908df434395b7a50e319a3085a6] to archive 2024-11-08T02:32:29,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:29,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a6136c2e77534c25987f0ef92e152004 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a6136c2e77534c25987f0ef92e152004 2024-11-08T02:32:29,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a35546371bc04e3c991961023616d879 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a35546371bc04e3c991961023616d879 2024-11-08T02:32:29,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/6de814f38d94412996baf2525b6b99e0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/6de814f38d94412996baf2525b6b99e0 2024-11-08T02:32:29,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d8816685a1fe4dbc908726c4fd946aa2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d8816685a1fe4dbc908726c4fd946aa2 2024-11-08T02:32:29,680 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/21606be01c294715a55f455f03793b06 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/21606be01c294715a55f455f03793b06 2024-11-08T02:32:29,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/cf02f29df538457c87094501be443b81 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/cf02f29df538457c87094501be443b81 2024-11-08T02:32:29,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/32be975829ee4fde8e64354a335dfadd to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/32be975829ee4fde8e64354a335dfadd 2024-11-08T02:32:29,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/5299437763cd42f293939bd034ba89b2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/5299437763cd42f293939bd034ba89b2 2024-11-08T02:32:29,685 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/0bcd0afde7b54885b3910886e659c7e3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/0bcd0afde7b54885b3910886e659c7e3 2024-11-08T02:32:29,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/358b8359fa2644408209a1c3330ed938 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/358b8359fa2644408209a1c3330ed938 2024-11-08T02:32:29,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/81e34ad68f3742769aad85574967527f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/81e34ad68f3742769aad85574967527f 2024-11-08T02:32:29,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/83b301020b894c859371064969d04150 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/83b301020b894c859371064969d04150 2024-11-08T02:32:29,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/9632f026082f4d41a7b4c419f4a8d55f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/9632f026082f4d41a7b4c419f4a8d55f 2024-11-08T02:32:29,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/35eb8680a42240489c10a9f841613c2c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/35eb8680a42240489c10a9f841613c2c 2024-11-08T02:32:29,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/ecdbff65922a:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a456c908df434395b7a50e319a3085a6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a456c908df434395b7a50e319a3085a6 2024-11-08T02:32:29,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a7c3c75f940a4b3b9367abf57b2b4a9b 2024-11-08T02:32:29,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/bac870cc4f36419588dd56e1a11cc343 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/bac870cc4f36419588dd56e1a11cc343 2024-11-08T02:32:29,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/bac870cc4f36419588dd56e1a11cc343, entries=200, sequenceid=255, filesize=38.8 K 2024-11-08T02:32:29,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/a7c98435cfd643ef8645195eff52a413 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a7c98435cfd643ef8645195eff52a413 2024-11-08T02:32:29,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a7c98435cfd643ef8645195eff52a413, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T02:32:29,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/a7c3c75f940a4b3b9367abf57b2b4a9b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a7c3c75f940a4b3b9367abf57b2b4a9b 2024-11-08T02:32:29,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a7c3c75f940a4b3b9367abf57b2b4a9b, entries=150, sequenceid=255, filesize=11.9 K 2024-11-08T02:32:29,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a2c847edb50c5b92217fdd99390ba5a5 in 893ms, sequenceid=255, compaction requested=true 2024-11-08T02:32:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:29,793 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a2c847edb50c5b92217fdd99390ba5a5:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:29,793 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:29,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:29,794 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:29,794 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:29,794 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/B is initiating minor compaction (all files) 2024-11-08T02:32:29,794 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/A is initiating minor compaction (all files) 2024-11-08T02:32:29,794 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/A in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:29,794 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/B in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:29,794 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/120ed9687d974910a48f5b868cb5ce96, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f8dfe51021ff4a3fb7625434588a1af9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/bac870cc4f36419588dd56e1a11cc343] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=100.1 K 2024-11-08T02:32:29,795 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a169c30ba42049fba0b635a3357ccb59, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32de285a41324b32b37c0fd7fcaeb933, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a7c98435cfd643ef8645195eff52a413] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=36.1 K 2024-11-08T02:32:29,795 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:29,795 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/120ed9687d974910a48f5b868cb5ce96, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f8dfe51021ff4a3fb7625434588a1af9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/bac870cc4f36419588dd56e1a11cc343] 2024-11-08T02:32:29,795 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a169c30ba42049fba0b635a3357ccb59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731033145823 2024-11-08T02:32:29,795 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 120ed9687d974910a48f5b868cb5ce96, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731033145823 2024-11-08T02:32:29,795 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f8dfe51021ff4a3fb7625434588a1af9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731033145883 2024-11-08T02:32:29,795 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 32de285a41324b32b37c0fd7fcaeb933, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731033145883 2024-11-08T02:32:29,796 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a7c98435cfd643ef8645195eff52a413, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731033147771 2024-11-08T02:32:29,796 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting bac870cc4f36419588dd56e1a11cc343, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731033147749 2024-11-08T02:32:29,801 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:29,802 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#B#compaction#303 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:29,802 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/07a25c27855445fe863aef0fae9c6e7f is 50, key is test_row_0/B:col10/1731033148899/Put/seqid=0 2024-11-08T02:32:29,803 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411087a1029b0c47849f0999d98712761e51e_a2c847edb50c5b92217fdd99390ba5a5 store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:29,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742190_1366 (size=12254) 2024-11-08T02:32:29,806 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411087a1029b0c47849f0999d98712761e51e_a2c847edb50c5b92217fdd99390ba5a5, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:29,806 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087a1029b0c47849f0999d98712761e51e_a2c847edb50c5b92217fdd99390ba5a5 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:29,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742191_1367 (size=4469) 2024-11-08T02:32:30,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:30,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:32:30,037 DEBUG [Thread-1358 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3dd5b441 to 127.0.0.1:56877 2024-11-08T02:32:30,037 DEBUG [Thread-1360 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c336ea4 to 127.0.0.1:56877 2024-11-08T02:32:30,037 DEBUG [Thread-1358 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:30,037 DEBUG [Thread-1360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:30,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:30,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:30,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:30,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:30,040 DEBUG [Thread-1354 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7b6cf8cb to 127.0.0.1:56877 2024-11-08T02:32:30,040 DEBUG [Thread-1354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:30,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108856601b923a245a79ea0f1dad6b55ae8_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_0/A:col10/1731033148983/Put/seqid=0 2024-11-08T02:32:30,045 DEBUG [Thread-1362 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f94d721 to 127.0.0.1:56877 2024-11-08T02:32:30,045 DEBUG [Thread-1362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:30,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742192_1368 (size=12454) 2024-11-08T02:32:30,107 DEBUG [Thread-1356 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ec15031 to 127.0.0.1:56877 2024-11-08T02:32:30,107 DEBUG [Thread-1356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-08T02:32:30,121 INFO [Thread-1364 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 110 completed 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2343 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7029 rows 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2336 2024-11-08T02:32:30,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7008 rows 2024-11-08T02:32:30,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2327 2024-11-08T02:32:30,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6981 rows 2024-11-08T02:32:30,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2343 2024-11-08T02:32:30,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7027 rows 2024-11-08T02:32:30,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2346 2024-11-08T02:32:30,122 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7038 rows 2024-11-08T02:32:30,122 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T02:32:30,122 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b82ba2a to 127.0.0.1:56877 2024-11-08T02:32:30,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:30,129 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-08T02:32:30,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:32:30,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:30,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-08T02:32:30,133 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033150133"}]},"ts":"1731033150133"} 2024-11-08T02:32:30,134 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:32:30,144 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:32:30,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:32:30,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, UNASSIGN}] 2024-11-08T02:32:30,146 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=114, ppid=113, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, UNASSIGN 2024-11-08T02:32:30,146 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=114 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:30,147 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:32:30,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; CloseRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:32:30,211 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:30,213 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/a43d4756fbe84a2bad5cebeb0137e470 is 175, key is test_row_0/A:col10/1731033148899/Put/seqid=0 2024-11-08T02:32:30,216 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/07a25c27855445fe863aef0fae9c6e7f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/07a25c27855445fe863aef0fae9c6e7f 2024-11-08T02:32:30,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742193_1369 (size=31208) 2024-11-08T02:32:30,222 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/B of a2c847edb50c5b92217fdd99390ba5a5 into 07a25c27855445fe863aef0fae9c6e7f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:30,222 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:30,222 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/B, priority=13, startTime=1731033149793; duration=0sec 2024-11-08T02:32:30,223 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:30,223 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:B 2024-11-08T02:32:30,223 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:30,224 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:30,224 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): a2c847edb50c5b92217fdd99390ba5a5/C is initiating minor compaction (all files) 2024-11-08T02:32:30,224 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of a2c847edb50c5b92217fdd99390ba5a5/C in TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:30,224 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/e65d3561a1024143b72caf727c0c65df, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/78e4d46c59fc4f4893bf446fb9d6adce, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a7c3c75f940a4b3b9367abf57b2b4a9b] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp, totalSize=36.1 K 2024-11-08T02:32:30,225 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e65d3561a1024143b72caf727c0c65df, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731033145823 2024-11-08T02:32:30,225 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 78e4d46c59fc4f4893bf446fb9d6adce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731033145883 2024-11-08T02:32:30,226 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a7c3c75f940a4b3b9367abf57b2b4a9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731033147771 2024-11-08T02:32:30,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-08T02:32:30,234 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a2c847edb50c5b92217fdd99390ba5a5#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:30,235 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/51c468f8064f46f093e58f579c0f320b is 50, key is test_row_0/C:col10/1731033148899/Put/seqid=0 2024-11-08T02:32:30,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742194_1370 (size=12254) 2024-11-08T02:32:30,299 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:30,299 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] handler.UnassignRegionHandler(124): Close a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:30,299 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:32:30,300 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1681): Closing a2c847edb50c5b92217fdd99390ba5a5, disabling compactions & flushes 2024-11-08T02:32:30,300 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1942): waiting for 2 compactions & cache flush to complete for region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:30,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-08T02:32:30,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:30,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108856601b923a245a79ea0f1dad6b55ae8_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108856601b923a245a79ea0f1dad6b55ae8_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:30,462 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/21253848ce6e4392b4ab80926f021f7b, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:30,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/21253848ce6e4392b4ab80926f021f7b is 175, key is test_row_0/A:col10/1731033148983/Put/seqid=0 2024-11-08T02:32:30,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742195_1371 (size=31255) 2024-11-08T02:32:30,628 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/a43d4756fbe84a2bad5cebeb0137e470 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/a43d4756fbe84a2bad5cebeb0137e470 2024-11-08T02:32:30,632 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/A of a2c847edb50c5b92217fdd99390ba5a5 into a43d4756fbe84a2bad5cebeb0137e470(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:30,632 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:30,632 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/A, priority=13, startTime=1731033149793; duration=0sec 2024-11-08T02:32:30,632 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:30,632 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:A 2024-11-08T02:32:30,643 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/51c468f8064f46f093e58f579c0f320b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/51c468f8064f46f093e58f579c0f320b 2024-11-08T02:32:30,647 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in a2c847edb50c5b92217fdd99390ba5a5/C of a2c847edb50c5b92217fdd99390ba5a5 into 51c468f8064f46f093e58f579c0f320b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:30,647 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:30,647 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5., storeName=a2c847edb50c5b92217fdd99390ba5a5/C, priority=13, startTime=1731033149793; duration=0sec 2024-11-08T02:32:30,647 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1942): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:30,647 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:30,647 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a2c847edb50c5b92217fdd99390ba5a5:C 2024-11-08T02:32:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-08T02:32:30,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/21253848ce6e4392b4ab80926f021f7b 2024-11-08T02:32:30,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/56a7d45c3f76475fb87f8a54ad40411e is 50, key is test_row_0/B:col10/1731033148983/Put/seqid=0 2024-11-08T02:32:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742196_1372 (size=12301) 2024-11-08T02:32:31,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-08T02:32:31,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/56a7d45c3f76475fb87f8a54ad40411e 2024-11-08T02:32:31,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/c160291324ee4e50a91f2941b0e90d9f is 50, key is test_row_0/C:col10/1731033148983/Put/seqid=0 2024-11-08T02:32:31,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742197_1373 (size=12301) 2024-11-08T02:32:31,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/c160291324ee4e50a91f2941b0e90d9f 2024-11-08T02:32:31,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/21253848ce6e4392b4ab80926f021f7b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/21253848ce6e4392b4ab80926f021f7b 2024-11-08T02:32:31,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/21253848ce6e4392b4ab80926f021f7b, entries=150, sequenceid=273, filesize=30.5 K 2024-11-08T02:32:31,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/56a7d45c3f76475fb87f8a54ad40411e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/56a7d45c3f76475fb87f8a54ad40411e 2024-11-08T02:32:31,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/56a7d45c3f76475fb87f8a54ad40411e, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:32:31,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/c160291324ee4e50a91f2941b0e90d9f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/c160291324ee4e50a91f2941b0e90d9f 2024-11-08T02:32:31,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/c160291324ee4e50a91f2941b0e90d9f, entries=150, sequenceid=273, filesize=12.0 K 2024-11-08T02:32:31,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=20.13 KB/20610 for a2c847edb50c5b92217fdd99390ba5a5 in 1690ms, sequenceid=273, compaction requested=false 2024-11-08T02:32:31,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:31,726 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. after waiting 0 ms 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:31,727 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(2837): Flushing a2c847edb50c5b92217fdd99390ba5a5 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=A 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=B 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a2c847edb50c5b92217fdd99390ba5a5, store=C 2024-11-08T02:32:31,727 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:31,733 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a83e5b962e3542ba9adf86e83e98d0ef_a2c847edb50c5b92217fdd99390ba5a5 is 50, key is test_row_1/A:col10/1731033150105/Put/seqid=0 2024-11-08T02:32:31,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742198_1374 (size=9914) 2024-11-08T02:32:32,137 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:32,139 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a83e5b962e3542ba9adf86e83e98d0ef_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a83e5b962e3542ba9adf86e83e98d0ef_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:32,140 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/5616cd23578a469784e5affb66a2f4d5, store: [table=TestAcidGuarantees family=A region=a2c847edb50c5b92217fdd99390ba5a5] 2024-11-08T02:32:32,141 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/5616cd23578a469784e5affb66a2f4d5 is 175, key is test_row_1/A:col10/1731033150105/Put/seqid=0 2024-11-08T02:32:32,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742199_1375 (size=22561) 2024-11-08T02:32:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-08T02:32:32,546 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/5616cd23578a469784e5affb66a2f4d5 2024-11-08T02:32:32,556 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/b507a8041c1a43f286649e47b2d7b290 is 50, key is test_row_1/B:col10/1731033150105/Put/seqid=0 2024-11-08T02:32:32,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742200_1376 (size=9857) 2024-11-08T02:32:32,962 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/b507a8041c1a43f286649e47b2d7b290 2024-11-08T02:32:32,977 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/d0ed0b58283947eda8079588846e461a is 50, key is test_row_1/C:col10/1731033150105/Put/seqid=0 2024-11-08T02:32:32,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742201_1377 (size=9857) 2024-11-08T02:32:33,382 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/d0ed0b58283947eda8079588846e461a 2024-11-08T02:32:33,391 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/A/5616cd23578a469784e5affb66a2f4d5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/5616cd23578a469784e5affb66a2f4d5 2024-11-08T02:32:33,395 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/5616cd23578a469784e5affb66a2f4d5, entries=100, sequenceid=282, filesize=22.0 K 2024-11-08T02:32:33,396 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/B/b507a8041c1a43f286649e47b2d7b290 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b507a8041c1a43f286649e47b2d7b290 2024-11-08T02:32:33,400 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b507a8041c1a43f286649e47b2d7b290, entries=100, sequenceid=282, filesize=9.6 K 2024-11-08T02:32:33,401 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/.tmp/C/d0ed0b58283947eda8079588846e461a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d0ed0b58283947eda8079588846e461a 2024-11-08T02:32:33,404 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d0ed0b58283947eda8079588846e461a, entries=100, sequenceid=282, filesize=9.6 K 2024-11-08T02:32:33,405 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(3040): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for a2c847edb50c5b92217fdd99390ba5a5 in 1678ms, sequenceid=282, compaction requested=true 2024-11-08T02:32:33,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/120ed9687d974910a48f5b868cb5ce96, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f8dfe51021ff4a3fb7625434588a1af9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/bac870cc4f36419588dd56e1a11cc343] to archive 2024-11-08T02:32:33,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:33,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/120ed9687d974910a48f5b868cb5ce96 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/120ed9687d974910a48f5b868cb5ce96 2024-11-08T02:32:33,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f8dfe51021ff4a3fb7625434588a1af9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/f8dfe51021ff4a3fb7625434588a1af9 2024-11-08T02:32:33,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/bac870cc4f36419588dd56e1a11cc343 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/bac870cc4f36419588dd56e1a11cc343 2024-11-08T02:32:33,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a169c30ba42049fba0b635a3357ccb59, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32de285a41324b32b37c0fd7fcaeb933, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a7c98435cfd643ef8645195eff52a413] to archive 2024-11-08T02:32:33,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:33,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a169c30ba42049fba0b635a3357ccb59 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a169c30ba42049fba0b635a3357ccb59 2024-11-08T02:32:33,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32de285a41324b32b37c0fd7fcaeb933 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/32de285a41324b32b37c0fd7fcaeb933 2024-11-08T02:32:33,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a7c98435cfd643ef8645195eff52a413 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/a7c98435cfd643ef8645195eff52a413 2024-11-08T02:32:33,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/e65d3561a1024143b72caf727c0c65df, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/78e4d46c59fc4f4893bf446fb9d6adce, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a7c3c75f940a4b3b9367abf57b2b4a9b] to archive 2024-11-08T02:32:33,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:32:33,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/e65d3561a1024143b72caf727c0c65df to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/e65d3561a1024143b72caf727c0c65df 2024-11-08T02:32:33,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/78e4d46c59fc4f4893bf446fb9d6adce to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/78e4d46c59fc4f4893bf446fb9d6adce 2024-11-08T02:32:33,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a7c3c75f940a4b3b9367abf57b2b4a9b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/a7c3c75f940a4b3b9367abf57b2b4a9b 2024-11-08T02:32:33,426 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/recovered.edits/285.seqid, newMaxSeqId=285, maxSeqId=4 2024-11-08T02:32:33,427 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5. 2024-11-08T02:32:33,427 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] regionserver.HRegion(1635): Region close journal for a2c847edb50c5b92217fdd99390ba5a5: 2024-11-08T02:32:33,428 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=115}] handler.UnassignRegionHandler(170): Closed a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:33,429 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=114 updating hbase:meta row=a2c847edb50c5b92217fdd99390ba5a5, regionState=CLOSED 2024-11-08T02:32:33,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-11-08T02:32:33,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; CloseRegionProcedure a2c847edb50c5b92217fdd99390ba5a5, server=ecdbff65922a,40577,1731033026611 in 3.2830 sec 2024-11-08T02:32:33,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=114, resume processing ppid=113 2024-11-08T02:32:33,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, ppid=113, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=a2c847edb50c5b92217fdd99390ba5a5, UNASSIGN in 3.2870 sec 2024-11-08T02:32:33,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=113, resume processing ppid=112 2024-11-08T02:32:33,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=113, ppid=112, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.2890 sec 2024-11-08T02:32:33,437 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033153437"}]},"ts":"1731033153437"} 2024-11-08T02:32:33,438 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:32:33,480 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:32:33,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=112, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 3.3520 sec 2024-11-08T02:32:34,055 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46319,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-08T02:32:34,243 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 112 completed 2024-11-08T02:32:34,245 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:32:34,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:34,249 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=116, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:34,251 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=116, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:34,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-08T02:32:34,255 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,259 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/recovered.edits] 2024-11-08T02:32:34,263 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/21253848ce6e4392b4ab80926f021f7b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/21253848ce6e4392b4ab80926f021f7b 2024-11-08T02:32:34,264 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/5616cd23578a469784e5affb66a2f4d5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/5616cd23578a469784e5affb66a2f4d5 2024-11-08T02:32:34,266 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/a43d4756fbe84a2bad5cebeb0137e470 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/A/a43d4756fbe84a2bad5cebeb0137e470 2024-11-08T02:32:34,269 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/07a25c27855445fe863aef0fae9c6e7f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/07a25c27855445fe863aef0fae9c6e7f 2024-11-08T02:32:34,270 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/56a7d45c3f76475fb87f8a54ad40411e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/56a7d45c3f76475fb87f8a54ad40411e 2024-11-08T02:32:34,272 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b507a8041c1a43f286649e47b2d7b290 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/B/b507a8041c1a43f286649e47b2d7b290 2024-11-08T02:32:34,276 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/51c468f8064f46f093e58f579c0f320b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/51c468f8064f46f093e58f579c0f320b 2024-11-08T02:32:34,277 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/c160291324ee4e50a91f2941b0e90d9f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/c160291324ee4e50a91f2941b0e90d9f 2024-11-08T02:32:34,279 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d0ed0b58283947eda8079588846e461a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/C/d0ed0b58283947eda8079588846e461a 2024-11-08T02:32:34,283 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/recovered.edits/285.seqid to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5/recovered.edits/285.seqid 2024-11-08T02:32:34,284 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,284 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:32:34,285 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:32:34,286 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T02:32:34,291 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110803cdbd01c1354620b41350cd14e0ad20_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110803cdbd01c1354620b41350cd14e0ad20_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,293 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080fb45b3acfa14148b9b8e2ad0451f85d_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080fb45b3acfa14148b9b8e2ad0451f85d_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,295 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083a3c0baf56da43b39eba66203ec01ec0_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083a3c0baf56da43b39eba66203ec01ec0_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,296 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108421cdd41f9294f96a8bec1928b63e814_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108421cdd41f9294f96a8bec1928b63e814_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,298 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845301f5821334ed4b280cab8743c27c2_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845301f5821334ed4b280cab8743c27c2_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,300 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084e35c434704d403db905497d03130b82_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084e35c434704d403db905497d03130b82_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,301 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110874f1a12c895242d7b656335cac59295b_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110874f1a12c895242d7b656335cac59295b_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,303 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108795c8d0a25344e1bb824d3a2a325d971_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108795c8d0a25344e1bb824d3a2a325d971_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,305 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882818d9bdc49469681f0bc8bb620a6b2_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110882818d9bdc49469681f0bc8bb620a6b2_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,307 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108856601b923a245a79ea0f1dad6b55ae8_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108856601b923a245a79ea0f1dad6b55ae8_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,309 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110895012848b0d64aedbc6af7367bace4b4_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110895012848b0d64aedbc6af7367bace4b4_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a83e5b962e3542ba9adf86e83e98d0ef_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a83e5b962e3542ba9adf86e83e98d0ef_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,312 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c5750ff5ccd445c3be27f1bbd846ee8e_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c5750ff5ccd445c3be27f1bbd846ee8e_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,315 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d034f0189aa14a89b89d18b233961058_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d034f0189aa14a89b89d18b233961058_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,317 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f1e96d9ff7144511b7d0b6e86cbe764b_a2c847edb50c5b92217fdd99390ba5a5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f1e96d9ff7144511b7d0b6e86cbe764b_a2c847edb50c5b92217fdd99390ba5a5 2024-11-08T02:32:34,317 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:32:34,321 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=116, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:34,323 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:32:34,326 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:32:34,327 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=116, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:34,327 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:32:34,327 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731033154327"}]},"ts":"9223372036854775807"} 2024-11-08T02:32:34,330 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-08T02:32:34,330 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => a2c847edb50c5b92217fdd99390ba5a5, NAME => 'TestAcidGuarantees,,1731033126061.a2c847edb50c5b92217fdd99390ba5a5.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:32:34,330 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:32:34,330 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731033154330"}]},"ts":"9223372036854775807"} 2024-11-08T02:32:34,332 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-08T02:32:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-08T02:32:34,373 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=116, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:34,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 128 msec 2024-11-08T02:32:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-08T02:32:34,554 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 116 completed 2024-11-08T02:32:34,568 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMobScanAtomicity Thread=238 (was 236) - Thread LEAK? -, OpenFileDescriptor=452 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2117 (was 2158) 2024-11-08T02:32:34,576 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testScanAtomicity Thread=238, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=2117 2024-11-08T02:32:34,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:32:34,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:32:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=117, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:32:34,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=117, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:32:34,579 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:34,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 117 2024-11-08T02:32:34,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=117, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:32:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=117 2024-11-08T02:32:34,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742202_1378 (size=960) 2024-11-08T02:32:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=117 2024-11-08T02:32:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=117 2024-11-08T02:32:34,993 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:32:35,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742203_1379 (size=53) 2024-11-08T02:32:35,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=117 2024-11-08T02:32:35,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:32:35,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing b16fc799559c1038c4d1f04e11c99eb6, disabling compactions & flushes 2024-11-08T02:32:35,403 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. after waiting 0 ms 2024-11-08T02:32:35,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,404 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,404 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:35,406 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=117, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:32:35,407 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731033155406"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731033155406"}]},"ts":"1731033155406"} 2024-11-08T02:32:35,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T02:32:35,411 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=117, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:32:35,411 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033155411"}]},"ts":"1731033155411"} 2024-11-08T02:32:35,412 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:32:35,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b16fc799559c1038c4d1f04e11c99eb6, ASSIGN}] 2024-11-08T02:32:35,466 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=118, ppid=117, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b16fc799559c1038c4d1f04e11c99eb6, ASSIGN 2024-11-08T02:32:35,468 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=118, ppid=117, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b16fc799559c1038c4d1f04e11c99eb6, ASSIGN; state=OFFLINE, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=false 2024-11-08T02:32:35,619 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=118 updating hbase:meta row=b16fc799559c1038c4d1f04e11c99eb6, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:35,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE; OpenRegionProcedure b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:32:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=117 2024-11-08T02:32:35,776 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:35,782 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,782 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(7285): Opening region: {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:32:35,783 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,783 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:32:35,783 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(7327): checking encryption for b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,783 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(7330): checking classloading for b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,787 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,790 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:35,790 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b16fc799559c1038c4d1f04e11c99eb6 columnFamilyName A 2024-11-08T02:32:35,790 DEBUG [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:35,791 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.HStore(327): Store=b16fc799559c1038c4d1f04e11c99eb6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:35,791 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,793 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:35,793 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b16fc799559c1038c4d1f04e11c99eb6 columnFamilyName B 2024-11-08T02:32:35,793 DEBUG [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:35,794 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.HStore(327): Store=b16fc799559c1038c4d1f04e11c99eb6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:35,794 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,795 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:32:35,795 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b16fc799559c1038c4d1f04e11c99eb6 columnFamilyName C 2024-11-08T02:32:35,795 DEBUG [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:32:35,796 INFO [StoreOpener-b16fc799559c1038c4d1f04e11c99eb6-1 {}] regionserver.HStore(327): Store=b16fc799559c1038c4d1f04e11c99eb6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:32:35,796 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,797 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,797 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,798 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:32:35,800 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(1085): writing seq id for b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:35,802 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:32:35,802 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(1102): Opened b16fc799559c1038c4d1f04e11c99eb6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71256518, jitterRate=0.06180486083030701}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:32:35,803 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegion(1001): Region open journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:35,803 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., pid=119, masterSystemTime=1731033155775 2024-11-08T02:32:35,804 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,804 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=119}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:35,805 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=118 updating hbase:meta row=b16fc799559c1038c4d1f04e11c99eb6, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:35,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=119, resume processing ppid=118 2024-11-08T02:32:35,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, ppid=118, state=SUCCESS; OpenRegionProcedure b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 in 184 msec 2024-11-08T02:32:35,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=118, resume processing ppid=117 2024-11-08T02:32:35,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=118, ppid=117, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=b16fc799559c1038c4d1f04e11c99eb6, ASSIGN in 343 msec 2024-11-08T02:32:35,808 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=117, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:32:35,808 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033155808"}]},"ts":"1731033155808"} 2024-11-08T02:32:35,809 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:32:35,856 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=117, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:32:35,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2780 sec 2024-11-08T02:32:36,070 ERROR [LeaseRenewer:jenkins@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46319,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=117 2024-11-08T02:32:36,690 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 117 completed 2024-11-08T02:32:36,693 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63cefe40 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32c12a30 2024-11-08T02:32:36,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b10416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,745 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37492, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,747 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:32:36,749 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48610, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:32:36,752 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65df2359 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5ef40578 2024-11-08T02:32:36,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f142b04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,764 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7d0ab200 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32bb71c 2024-11-08T02:32:36,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de9f076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,773 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5871c039 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bc0f7c 2024-11-08T02:32:36,780 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4414259d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,781 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7daa5922 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b8b6e04 2024-11-08T02:32:36,789 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ed69825, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,789 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b7f20c4 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5bc486e1 2024-11-08T02:32:36,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11193a0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,798 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2070263a to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7861b162 2024-11-08T02:32:36,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf40102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,807 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6050584c to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@154f0f85 2024-11-08T02:32:36,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@496fe03f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,814 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6dd48863 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8a917b 2024-11-08T02:32:36,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3652e74d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,823 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51196534 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54c2725 2024-11-08T02:32:36,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2405c04e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,831 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1dc5e114 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79d49886 2024-11-08T02:32:36,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d92042, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:32:36,840 DEBUG [hconnection-0x6d64226c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:36,841 DEBUG [hconnection-0x7bf9e75e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=120, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees 2024-11-08T02:32:36,841 DEBUG [hconnection-0x692fd1c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,841 DEBUG [hconnection-0x68b93db-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,842 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37504, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-11-08T02:32:36,842 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37518, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,842 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37520, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,842 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37530, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,842 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=120, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:36,843 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=120, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:36,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:36,844 DEBUG [hconnection-0x1be57cf7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,844 DEBUG [hconnection-0x683e8376-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,845 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37546, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,845 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37560, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,847 DEBUG [hconnection-0x40af2447-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,847 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37570, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:36,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:32:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:36,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:36,851 DEBUG [hconnection-0x6364c33c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,851 DEBUG [hconnection-0x5c3e5595-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,852 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37574, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,852 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37578, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033216857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033216857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033216858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033216858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,863 DEBUG [hconnection-0x2e3232fb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:32:36,864 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37582, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:32:36,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033216865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/1e85bc2b30c54b29809509cc2e7b3db2 is 50, key is test_row_0/A:col10/1731033156848/Put/seqid=0 2024-11-08T02:32:36,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742204_1380 (size=12001) 2024-11-08T02:32:36,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/1e85bc2b30c54b29809509cc2e7b3db2 2024-11-08T02:32:36,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6f32dc3cd4514e02b204e78bf07e9472 is 50, key is test_row_0/B:col10/1731033156848/Put/seqid=0 2024-11-08T02:32:36,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742205_1381 (size=12001) 2024-11-08T02:32:36,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6f32dc3cd4514e02b204e78bf07e9472 2024-11-08T02:32:36,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-11-08T02:32:36,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/543818838e2d4db8bd496c53409f5af1 is 50, key is test_row_0/C:col10/1731033156848/Put/seqid=0 2024-11-08T02:32:36,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742206_1382 (size=12001) 2024-11-08T02:32:36,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/543818838e2d4db8bd496c53409f5af1 2024-11-08T02:32:36,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033216959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033216959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033216959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033216959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/1e85bc2b30c54b29809509cc2e7b3db2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1e85bc2b30c54b29809509cc2e7b3db2 2024-11-08T02:32:36,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033216969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1e85bc2b30c54b29809509cc2e7b3db2, entries=150, sequenceid=14, filesize=11.7 K 2024-11-08T02:32:36,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6f32dc3cd4514e02b204e78bf07e9472 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6f32dc3cd4514e02b204e78bf07e9472 2024-11-08T02:32:36,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6f32dc3cd4514e02b204e78bf07e9472, entries=150, sequenceid=14, filesize=11.7 K 2024-11-08T02:32:36,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/543818838e2d4db8bd496c53409f5af1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/543818838e2d4db8bd496c53409f5af1 2024-11-08T02:32:36,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/543818838e2d4db8bd496c53409f5af1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-08T02:32:36,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for b16fc799559c1038c4d1f04e11c99eb6 in 131ms, sequenceid=14, compaction requested=false 2024-11-08T02:32:36,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:36,994 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:36,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=121 2024-11-08T02:32:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:36,995 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:32:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:36,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:36,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/493dcd911b4f4458aca70988e4936ff8 is 50, key is test_row_0/A:col10/1731033156856/Put/seqid=0 2024-11-08T02:32:37,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742207_1383 (size=12001) 2024-11-08T02:32:37,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-11-08T02:32:37,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:37,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033217169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033217171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033217171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033217171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033217172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033217274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033217274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033217274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033217277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,403 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/493dcd911b4f4458aca70988e4936ff8 2024-11-08T02:32:37,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/18f098f598824145a526dddc011f742a is 50, key is test_row_0/B:col10/1731033156856/Put/seqid=0 2024-11-08T02:32:37,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742208_1384 (size=12001) 2024-11-08T02:32:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-11-08T02:32:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033217477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033217477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033217477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033217479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033217481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033217781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033217783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033217783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033217784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:37,813 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/18f098f598824145a526dddc011f742a 2024-11-08T02:32:37,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/0fe325045cb64dfd8df8173b7b9ed6f3 is 50, key is test_row_0/C:col10/1731033156856/Put/seqid=0 2024-11-08T02:32:37,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742209_1385 (size=12001) 2024-11-08T02:32:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-11-08T02:32:37,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033217981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,223 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/0fe325045cb64dfd8df8173b7b9ed6f3 2024-11-08T02:32:38,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/493dcd911b4f4458aca70988e4936ff8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/493dcd911b4f4458aca70988e4936ff8 2024-11-08T02:32:38,230 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/493dcd911b4f4458aca70988e4936ff8, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T02:32:38,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/18f098f598824145a526dddc011f742a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/18f098f598824145a526dddc011f742a 2024-11-08T02:32:38,234 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/18f098f598824145a526dddc011f742a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T02:32:38,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/0fe325045cb64dfd8df8173b7b9ed6f3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0fe325045cb64dfd8df8173b7b9ed6f3 2024-11-08T02:32:38,238 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0fe325045cb64dfd8df8173b7b9ed6f3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T02:32:38,239 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b16fc799559c1038c4d1f04e11c99eb6 in 1245ms, sequenceid=38, compaction requested=false 2024-11-08T02:32:38,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:38,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=121}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=121 2024-11-08T02:32:38,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=121 2024-11-08T02:32:38,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=121, resume processing ppid=120 2024-11-08T02:32:38,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, ppid=120, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3970 sec 2024-11-08T02:32:38,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=120, table=TestAcidGuarantees in 1.4000 sec 2024-11-08T02:32:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:38,290 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:32:38,290 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:38,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:38,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:38,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:38,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:38,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:38,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/bd4eb8f758b14b799528069b4e616925 is 50, key is test_row_0/A:col10/1731033158290/Put/seqid=0 2024-11-08T02:32:38,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742210_1386 (size=9657) 2024-11-08T02:32:38,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/bd4eb8f758b14b799528069b4e616925 2024-11-08T02:32:38,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6a2073863ae84927b67ec21756c2e793 is 50, key is test_row_0/B:col10/1731033158290/Put/seqid=0 2024-11-08T02:32:38,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742211_1387 (size=9657) 2024-11-08T02:32:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033218319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033218322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033218322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033218323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033218427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033218427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033218433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033218433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033218632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033218632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033218637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033218638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6a2073863ae84927b67ec21756c2e793 2024-11-08T02:32:38,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/4f833124d4374441a57bde8dc570f9d6 is 50, key is test_row_0/C:col10/1731033158290/Put/seqid=0 2024-11-08T02:32:38,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742212_1388 (size=9657) 2024-11-08T02:32:38,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033218935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033218937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033218943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=120 2024-11-08T02:32:38,945 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 120 completed 2024-11-08T02:32:38,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=122, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees 2024-11-08T02:32:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-08T02:32:38,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=122, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:38,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,947 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=122, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:38,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033218943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:38,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:38,949 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:32:38,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033218985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-08T02:32:39,098 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-08T02:32:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:39,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:39,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:39,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/4f833124d4374441a57bde8dc570f9d6 2024-11-08T02:32:39,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/bd4eb8f758b14b799528069b4e616925 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/bd4eb8f758b14b799528069b4e616925 2024-11-08T02:32:39,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/bd4eb8f758b14b799528069b4e616925, entries=100, sequenceid=52, filesize=9.4 K 2024-11-08T02:32:39,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6a2073863ae84927b67ec21756c2e793 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6a2073863ae84927b67ec21756c2e793 2024-11-08T02:32:39,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6a2073863ae84927b67ec21756c2e793, entries=100, sequenceid=52, filesize=9.4 K 2024-11-08T02:32:39,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/4f833124d4374441a57bde8dc570f9d6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/4f833124d4374441a57bde8dc570f9d6 2024-11-08T02:32:39,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/4f833124d4374441a57bde8dc570f9d6, entries=100, sequenceid=52, filesize=9.4 K 2024-11-08T02:32:39,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b16fc799559c1038c4d1f04e11c99eb6 in 862ms, sequenceid=52, compaction requested=true 2024-11-08T02:32:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:39,151 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:39,151 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:39,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:39,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:39,152 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:39,152 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:39,152 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:39,152 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:39,152 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:39,152 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:39,152 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1e85bc2b30c54b29809509cc2e7b3db2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/493dcd911b4f4458aca70988e4936ff8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/bd4eb8f758b14b799528069b4e616925] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=32.9 K 2024-11-08T02:32:39,152 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6f32dc3cd4514e02b204e78bf07e9472, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/18f098f598824145a526dddc011f742a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6a2073863ae84927b67ec21756c2e793] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=32.9 K 2024-11-08T02:32:39,153 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6f32dc3cd4514e02b204e78bf07e9472, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731033156847 2024-11-08T02:32:39,153 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1e85bc2b30c54b29809509cc2e7b3db2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731033156847 2024-11-08T02:32:39,153 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 18f098f598824145a526dddc011f742a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731033156855 2024-11-08T02:32:39,153 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 493dcd911b4f4458aca70988e4936ff8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731033156855 2024-11-08T02:32:39,153 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6a2073863ae84927b67ec21756c2e793, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731033158288 2024-11-08T02:32:39,153 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting bd4eb8f758b14b799528069b4e616925, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731033158288 2024-11-08T02:32:39,158 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:39,158 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/1c957260a38848349d01ec9b94a09d00 is 50, key is test_row_0/A:col10/1731033158290/Put/seqid=0 2024-11-08T02:32:39,158 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:39,159 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/b7fd0be761b944ea9e86f48888ed1c1e is 50, key is test_row_0/B:col10/1731033158290/Put/seqid=0 2024-11-08T02:32:39,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742213_1389 (size=12104) 2024-11-08T02:32:39,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742214_1390 (size=12104) 2024-11-08T02:32:39,170 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/b7fd0be761b944ea9e86f48888ed1c1e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b7fd0be761b944ea9e86f48888ed1c1e 2024-11-08T02:32:39,171 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/1c957260a38848349d01ec9b94a09d00 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1c957260a38848349d01ec9b94a09d00 2024-11-08T02:32:39,174 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into 1c957260a38848349d01ec9b94a09d00(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:39,174 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into b7fd0be761b944ea9e86f48888ed1c1e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:39,174 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:39,174 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:39,174 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=13, startTime=1731033159151; duration=0sec 2024-11-08T02:32:39,174 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=13, startTime=1731033159151; duration=0sec 2024-11-08T02:32:39,174 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:39,174 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:39,174 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:39,174 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:39,174 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:39,175 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:39,175 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:39,175 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:39,176 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/543818838e2d4db8bd496c53409f5af1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0fe325045cb64dfd8df8173b7b9ed6f3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/4f833124d4374441a57bde8dc570f9d6] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=32.9 K 2024-11-08T02:32:39,176 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 543818838e2d4db8bd496c53409f5af1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731033156847 2024-11-08T02:32:39,176 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0fe325045cb64dfd8df8173b7b9ed6f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731033156855 2024-11-08T02:32:39,176 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4f833124d4374441a57bde8dc570f9d6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731033158288 2024-11-08T02:32:39,181 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#323 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:39,181 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/3a31c1f0d58b4752956aa28d5874b663 is 50, key is test_row_0/C:col10/1731033158290/Put/seqid=0 2024-11-08T02:32:39,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742215_1391 (size=12104) 2024-11-08T02:32:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-08T02:32:39,251 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-08T02:32:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:39,251 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:32:39,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:39,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:39,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/edb682603db544ac86cc2461c67f2065 is 50, key is test_row_0/A:col10/1731033158322/Put/seqid=0 2024-11-08T02:32:39,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742216_1392 (size=12001) 2024-11-08T02:32:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:39,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033219456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033219459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033219460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033219461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-08T02:32:39,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033219562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033219562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033219566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033219566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,591 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/3a31c1f0d58b4752956aa28d5874b663 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3a31c1f0d58b4752956aa28d5874b663 2024-11-08T02:32:39,595 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into 3a31c1f0d58b4752956aa28d5874b663(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:39,596 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:39,596 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=13, startTime=1731033159151; duration=0sec 2024-11-08T02:32:39,596 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:39,596 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:39,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/edb682603db544ac86cc2461c67f2065 2024-11-08T02:32:39,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/90fd399fe82a44b3bad1cef4d0c1f66e is 50, key is test_row_0/B:col10/1731033158322/Put/seqid=0 2024-11-08T02:32:39,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742217_1393 (size=12001) 2024-11-08T02:32:39,672 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/90fd399fe82a44b3bad1cef4d0c1f66e 2024-11-08T02:32:39,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/eed68c339a054b6f9da429ef770c4f27 is 50, key is test_row_0/C:col10/1731033158322/Put/seqid=0 2024-11-08T02:32:39,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742218_1394 (size=12001) 2024-11-08T02:32:39,684 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/eed68c339a054b6f9da429ef770c4f27 2024-11-08T02:32:39,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/edb682603db544ac86cc2461c67f2065 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/edb682603db544ac86cc2461c67f2065 2024-11-08T02:32:39,692 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/edb682603db544ac86cc2461c67f2065, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:32:39,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/90fd399fe82a44b3bad1cef4d0c1f66e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/90fd399fe82a44b3bad1cef4d0c1f66e 2024-11-08T02:32:39,697 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/90fd399fe82a44b3bad1cef4d0c1f66e, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:32:39,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/eed68c339a054b6f9da429ef770c4f27 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/eed68c339a054b6f9da429ef770c4f27 2024-11-08T02:32:39,703 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/eed68c339a054b6f9da429ef770c4f27, entries=150, sequenceid=76, filesize=11.7 K 2024-11-08T02:32:39,704 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for b16fc799559c1038c4d1f04e11c99eb6 in 452ms, sequenceid=76, compaction requested=false 2024-11-08T02:32:39,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:39,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:39,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=123 2024-11-08T02:32:39,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=123 2024-11-08T02:32:39,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=123, resume processing ppid=122 2024-11-08T02:32:39,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, ppid=122, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 758 msec 2024-11-08T02:32:39,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees in 761 msec 2024-11-08T02:32:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:39,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T02:32:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:39,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/53e6002783294dbbb05405054b9f591a is 50, key is test_row_0/A:col10/1731033159771/Put/seqid=0 2024-11-08T02:32:39,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742219_1395 (size=14341) 2024-11-08T02:32:39,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033219801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033219802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033219807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033219807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033219908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033219912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033219914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:39,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033219914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-08T02:32:40,050 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 122 completed 2024-11-08T02:32:40,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=124, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees 2024-11-08T02:32:40,052 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=124, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-08T02:32:40,052 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=124, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:40,052 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:40,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033220114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033220119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033220124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033220124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-08T02:32:40,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/53e6002783294dbbb05405054b9f591a 2024-11-08T02:32:40,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/a986152e5e17450f9de838388fbf653b is 50, key is test_row_0/B:col10/1731033159771/Put/seqid=0 2024-11-08T02:32:40,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742220_1396 (size=12001) 2024-11-08T02:32:40,203 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-08T02:32:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:40,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-08T02:32:40,355 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-08T02:32:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033220418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033220424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033220430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033220430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,507 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-08T02:32:40,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:40,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/a986152e5e17450f9de838388fbf653b 2024-11-08T02:32:40,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/308344520ff441b5b16b15b90fcdcc55 is 50, key is test_row_0/C:col10/1731033159771/Put/seqid=0 2024-11-08T02:32:40,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742221_1397 (size=12001) 2024-11-08T02:32:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-08T02:32:40,660 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-08T02:32:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,812 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-08T02:32:40,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033220923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033220928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033220933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033220934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,965 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-08T02:32:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:40,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:40,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033220993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:40,998 DEBUG [Thread-1730 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4158 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:41,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/308344520ff441b5b16b15b90fcdcc55 2024-11-08T02:32:41,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/53e6002783294dbbb05405054b9f591a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/53e6002783294dbbb05405054b9f591a 2024-11-08T02:32:41,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/53e6002783294dbbb05405054b9f591a, entries=200, sequenceid=93, filesize=14.0 K 2024-11-08T02:32:41,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/a986152e5e17450f9de838388fbf653b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a986152e5e17450f9de838388fbf653b 2024-11-08T02:32:41,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a986152e5e17450f9de838388fbf653b, entries=150, sequenceid=93, filesize=11.7 K 2024-11-08T02:32:41,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/308344520ff441b5b16b15b90fcdcc55 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/308344520ff441b5b16b15b90fcdcc55 2024-11-08T02:32:41,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/308344520ff441b5b16b15b90fcdcc55, entries=150, sequenceid=93, filesize=11.7 K 2024-11-08T02:32:41,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for b16fc799559c1038c4d1f04e11c99eb6 in 1245ms, sequenceid=93, compaction requested=true 2024-11-08T02:32:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:41,017 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:41,017 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:41,018 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:41,018 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:41,018 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:41,018 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:41,018 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:41,018 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:41,018 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b7fd0be761b944ea9e86f48888ed1c1e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/90fd399fe82a44b3bad1cef4d0c1f66e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a986152e5e17450f9de838388fbf653b] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=35.3 K 2024-11-08T02:32:41,018 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1c957260a38848349d01ec9b94a09d00, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/edb682603db544ac86cc2461c67f2065, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/53e6002783294dbbb05405054b9f591a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=37.5 K 2024-11-08T02:32:41,018 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1c957260a38848349d01ec9b94a09d00, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731033156855 2024-11-08T02:32:41,018 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b7fd0be761b944ea9e86f48888ed1c1e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731033156855 2024-11-08T02:32:41,019 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting edb682603db544ac86cc2461c67f2065, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033158317 2024-11-08T02:32:41,019 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 90fd399fe82a44b3bad1cef4d0c1f66e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033158317 2024-11-08T02:32:41,019 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 53e6002783294dbbb05405054b9f591a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731033159459 2024-11-08T02:32:41,019 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a986152e5e17450f9de838388fbf653b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731033159460 2024-11-08T02:32:41,026 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:41,026 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:41,026 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/38797736308b4f0fb2b1164e01d98e79 is 50, key is test_row_0/A:col10/1731033159771/Put/seqid=0 2024-11-08T02:32:41,026 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/686c9151c6134112bcdc2ea8d9fd536b is 50, key is test_row_0/B:col10/1731033159771/Put/seqid=0 2024-11-08T02:32:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742222_1398 (size=12207) 2024-11-08T02:32:41,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742223_1399 (size=12207) 2024-11-08T02:32:41,034 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/686c9151c6134112bcdc2ea8d9fd536b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/686c9151c6134112bcdc2ea8d9fd536b 2024-11-08T02:32:41,037 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into 686c9151c6134112bcdc2ea8d9fd536b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:41,037 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:41,037 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=13, startTime=1731033161017; duration=0sec 2024-11-08T02:32:41,037 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:41,037 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:41,037 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:41,038 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:41,038 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:41,038 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:41,038 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3a31c1f0d58b4752956aa28d5874b663, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/eed68c339a054b6f9da429ef770c4f27, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/308344520ff441b5b16b15b90fcdcc55] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=35.3 K 2024-11-08T02:32:41,038 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 3a31c1f0d58b4752956aa28d5874b663, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731033156855 2024-11-08T02:32:41,039 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting eed68c339a054b6f9da429ef770c4f27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731033158317 2024-11-08T02:32:41,039 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 308344520ff441b5b16b15b90fcdcc55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731033159460 2024-11-08T02:32:41,044 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:41,044 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/81e01757b00b4994bb1caab50426596e is 50, key is test_row_0/C:col10/1731033159771/Put/seqid=0 2024-11-08T02:32:41,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742224_1400 (size=12207) 2024-11-08T02:32:41,117 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:41,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-08T02:32:41,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:41,118 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T02:32:41,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:41,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:41,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:41,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:41,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d is 50, key is test_row_0/A:col10/1731033159806/Put/seqid=0 2024-11-08T02:32:41,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742225_1401 (size=12001) 2024-11-08T02:32:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-08T02:32:41,433 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/38797736308b4f0fb2b1164e01d98e79 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/38797736308b4f0fb2b1164e01d98e79 2024-11-08T02:32:41,436 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into 38797736308b4f0fb2b1164e01d98e79(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:41,436 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:41,436 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=13, startTime=1731033161017; duration=0sec 2024-11-08T02:32:41,437 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:41,437 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:41,451 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/81e01757b00b4994bb1caab50426596e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/81e01757b00b4994bb1caab50426596e 2024-11-08T02:32:41,454 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into 81e01757b00b4994bb1caab50426596e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:41,454 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:41,454 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=13, startTime=1731033161017; duration=0sec 2024-11-08T02:32:41,454 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:41,454 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:41,526 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d 2024-11-08T02:32:41,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/5effc6cbe93b49f891b1d4d71b16e66f is 50, key is test_row_0/B:col10/1731033159806/Put/seqid=0 2024-11-08T02:32:41,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742226_1402 (size=12001) 2024-11-08T02:32:41,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:41,950 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/5effc6cbe93b49f891b1d4d71b16e66f 2024-11-08T02:32:41,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/5f26b310ff01416b96e3e368c87079b4 is 50, key is test_row_0/C:col10/1731033159806/Put/seqid=0 2024-11-08T02:32:41,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033221952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:41,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:41,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033221952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033221956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:41,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033221956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:41,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742227_1403 (size=12001) 2024-11-08T02:32:41,964 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/5f26b310ff01416b96e3e368c87079b4 2024-11-08T02:32:41,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d 2024-11-08T02:32:41,970 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:32:41,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/5effc6cbe93b49f891b1d4d71b16e66f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/5effc6cbe93b49f891b1d4d71b16e66f 2024-11-08T02:32:41,974 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/5effc6cbe93b49f891b1d4d71b16e66f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:32:41,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/5f26b310ff01416b96e3e368c87079b4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/5f26b310ff01416b96e3e368c87079b4 2024-11-08T02:32:41,978 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/5f26b310ff01416b96e3e368c87079b4, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T02:32:41,979 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b16fc799559c1038c4d1f04e11c99eb6 in 861ms, sequenceid=115, compaction requested=false 2024-11-08T02:32:41,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:41,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:41,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=125 2024-11-08T02:32:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=125 2024-11-08T02:32:41,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=125, resume processing ppid=124 2024-11-08T02:32:41,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=125, ppid=124, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9280 sec 2024-11-08T02:32:41,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees in 1.9300 sec 2024-11-08T02:32:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:42,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T02:32:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:42,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0a48931e5b2a456f8f397746b69b23bc is 50, key is test_row_0/A:col10/1731033162059/Put/seqid=0 2024-11-08T02:32:42,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742228_1404 (size=16881) 2024-11-08T02:32:42,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033222084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033222085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033222090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033222091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-08T02:32:42,155 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 124 completed 2024-11-08T02:32:42,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees 2024-11-08T02:32:42,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=126, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-08T02:32:42,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=126, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:42,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:42,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033222195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033222195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033222197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033222198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-08T02:32:42,308 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:42,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:42,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033222400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033222400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033222401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033222404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-08T02:32:42,461 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0a48931e5b2a456f8f397746b69b23bc 2024-11-08T02:32:42,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/ffb8b1f9f89c4c349b4c373f85a728b2 is 50, key is test_row_0/B:col10/1731033162059/Put/seqid=0 2024-11-08T02:32:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742229_1405 (size=12151) 2024-11-08T02:32:42,613 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:42,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:42,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033222703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033222703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033222704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033222711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-08T02:32:42,765 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/ffb8b1f9f89c4c349b4c373f85a728b2 2024-11-08T02:32:42,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/07a9c6d46d94475ba8b7662abdfd9c02 is 50, key is test_row_0/C:col10/1731033162059/Put/seqid=0 2024-11-08T02:32:42,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742230_1406 (size=12151) 2024-11-08T02:32:42,918 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:42,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:42,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:42,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:42,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:43,070 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:43,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:43,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:43,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:43,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:43,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:43,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033223211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:43,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033223212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:43,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:43,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033223212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:43,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033223215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:43,222 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:43,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:43,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:43,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:43,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:43,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:43,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-08T02:32:43,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/07a9c6d46d94475ba8b7662abdfd9c02 2024-11-08T02:32:43,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0a48931e5b2a456f8f397746b69b23bc as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0a48931e5b2a456f8f397746b69b23bc 2024-11-08T02:32:43,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0a48931e5b2a456f8f397746b69b23bc, entries=250, sequenceid=134, filesize=16.5 K 2024-11-08T02:32:43,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/ffb8b1f9f89c4c349b4c373f85a728b2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ffb8b1f9f89c4c349b4c373f85a728b2 2024-11-08T02:32:43,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ffb8b1f9f89c4c349b4c373f85a728b2, entries=150, sequenceid=134, filesize=11.9 K 2024-11-08T02:32:43,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/07a9c6d46d94475ba8b7662abdfd9c02 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/07a9c6d46d94475ba8b7662abdfd9c02 2024-11-08T02:32:43,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/07a9c6d46d94475ba8b7662abdfd9c02, entries=150, sequenceid=134, filesize=11.9 K 2024-11-08T02:32:43,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b16fc799559c1038c4d1f04e11c99eb6 in 1241ms, sequenceid=134, compaction requested=true 2024-11-08T02:32:43,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:43,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:43,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:43,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:43,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:43,302 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:43,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:43,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:43,302 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:43,302 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:43,302 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:43,302 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:43,302 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:43,302 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,302 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,302 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/38797736308b4f0fb2b1164e01d98e79, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0a48931e5b2a456f8f397746b69b23bc] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=40.1 K 2024-11-08T02:32:43,302 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/686c9151c6134112bcdc2ea8d9fd536b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/5effc6cbe93b49f891b1d4d71b16e66f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ffb8b1f9f89c4c349b4c373f85a728b2] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=35.5 K 2024-11-08T02:32:43,303 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 686c9151c6134112bcdc2ea8d9fd536b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731033159460 2024-11-08T02:32:43,303 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 38797736308b4f0fb2b1164e01d98e79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731033159460 2024-11-08T02:32:43,303 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ec497c4ef2ec4fbcbcecb93bfe12bd8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033159794 2024-11-08T02:32:43,303 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 5effc6cbe93b49f891b1d4d71b16e66f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033159794 2024-11-08T02:32:43,303 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0a48931e5b2a456f8f397746b69b23bc, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731033161950 2024-11-08T02:32:43,304 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting ffb8b1f9f89c4c349b4c373f85a728b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731033161955 2024-11-08T02:32:43,309 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:43,309 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/20c441f326a04bb7a76af34180e9f0c6 is 50, key is test_row_0/B:col10/1731033162059/Put/seqid=0 2024-11-08T02:32:43,310 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:43,310 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0db2c27c976940c8821148256d694db1 is 50, key is test_row_0/A:col10/1731033162059/Put/seqid=0 2024-11-08T02:32:43,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742231_1407 (size=12459) 2024-11-08T02:32:43,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742232_1408 (size=12459) 2024-11-08T02:32:43,374 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:43,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-08T02:32:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,375 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T02:32:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:43,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:43,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c9118afefa044d75b44ebf7127a7c4e6 is 50, key is test_row_0/A:col10/1731033162090/Put/seqid=0 2024-11-08T02:32:43,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742233_1409 (size=12151) 2024-11-08T02:32:43,381 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c9118afefa044d75b44ebf7127a7c4e6 2024-11-08T02:32:43,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/13a976792d6c4eb8839c93a83e2c344d is 50, key is test_row_0/B:col10/1731033162090/Put/seqid=0 2024-11-08T02:32:43,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742234_1410 (size=12151) 2024-11-08T02:32:43,722 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/20c441f326a04bb7a76af34180e9f0c6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/20c441f326a04bb7a76af34180e9f0c6 2024-11-08T02:32:43,726 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into 20c441f326a04bb7a76af34180e9f0c6(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:43,726 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:43,726 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=13, startTime=1731033163301; duration=0sec 2024-11-08T02:32:43,726 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:43,726 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:43,726 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:43,727 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:43,727 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:43,727 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,727 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/81e01757b00b4994bb1caab50426596e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/5f26b310ff01416b96e3e368c87079b4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/07a9c6d46d94475ba8b7662abdfd9c02] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=35.5 K 2024-11-08T02:32:43,727 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 81e01757b00b4994bb1caab50426596e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731033159460 2024-11-08T02:32:43,727 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 5f26b310ff01416b96e3e368c87079b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731033159794 2024-11-08T02:32:43,728 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 07a9c6d46d94475ba8b7662abdfd9c02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731033161955 2024-11-08T02:32:43,728 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0db2c27c976940c8821148256d694db1 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0db2c27c976940c8821148256d694db1 2024-11-08T02:32:43,733 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into 0db2c27c976940c8821148256d694db1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:43,733 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:43,733 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=13, startTime=1731033163301; duration=0sec 2024-11-08T02:32:43,733 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:43,733 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:43,735 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:43,736 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/7c061783bd0847f48a94dc8c45afd696 is 50, key is test_row_0/C:col10/1731033162059/Put/seqid=0 2024-11-08T02:32:43,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742235_1411 (size=12459) 2024-11-08T02:32:43,790 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/13a976792d6c4eb8839c93a83e2c344d 2024-11-08T02:32:43,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/d1892fdf2a54469b965d0c61ef2a9ad0 is 50, key is test_row_0/C:col10/1731033162090/Put/seqid=0 2024-11-08T02:32:43,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742236_1412 (size=12151) 2024-11-08T02:32:43,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/d1892fdf2a54469b965d0c61ef2a9ad0 2024-11-08T02:32:43,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c9118afefa044d75b44ebf7127a7c4e6 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c9118afefa044d75b44ebf7127a7c4e6 2024-11-08T02:32:43,805 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c9118afefa044d75b44ebf7127a7c4e6, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T02:32:43,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/13a976792d6c4eb8839c93a83e2c344d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/13a976792d6c4eb8839c93a83e2c344d 2024-11-08T02:32:43,808 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/13a976792d6c4eb8839c93a83e2c344d, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T02:32:43,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/d1892fdf2a54469b965d0c61ef2a9ad0 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d1892fdf2a54469b965d0c61ef2a9ad0 2024-11-08T02:32:43,812 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d1892fdf2a54469b965d0c61ef2a9ad0, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T02:32:43,812 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for b16fc799559c1038c4d1f04e11c99eb6 in 437ms, sequenceid=153, compaction requested=false 2024-11-08T02:32:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:43,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=127 2024-11-08T02:32:43,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=127 2024-11-08T02:32:43,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=127, resume processing ppid=126 2024-11-08T02:32:43,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, ppid=126, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6550 sec 2024-11-08T02:32:43,815 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=126, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees in 1.6580 sec 2024-11-08T02:32:44,142 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/7c061783bd0847f48a94dc8c45afd696 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/7c061783bd0847f48a94dc8c45afd696 2024-11-08T02:32:44,145 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into 7c061783bd0847f48a94dc8c45afd696(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:44,145 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:44,145 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=13, startTime=1731033163302; duration=0sec 2024-11-08T02:32:44,146 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:44,146 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:44,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:44,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:32:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:44,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:44,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c0ad0a223a8649309f7964265650bb84 is 50, key is test_row_0/A:col10/1731033164226/Put/seqid=0 2024-11-08T02:32:44,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742237_1413 (size=14541) 2024-11-08T02:32:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-08T02:32:44,260 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 126 completed 2024-11-08T02:32:44,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees 2024-11-08T02:32:44,262 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:44,263 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:44,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-08T02:32:44,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033224259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033224260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033224267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033224267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-08T02:32:44,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033224368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033224368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033224375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033224375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,414 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:44,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-08T02:32:44,566 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033224572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033224573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033224581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033224581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c0ad0a223a8649309f7964265650bb84 2024-11-08T02:32:44,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/86584502c94d4859af1d5db4d13e630f is 50, key is test_row_0/B:col10/1731033164226/Put/seqid=0 2024-11-08T02:32:44,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742238_1414 (size=12151) 2024-11-08T02:32:44,719 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-08T02:32:44,871 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:44,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:44,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033224878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033224879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033224885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:44,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033224887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033225008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,014 DEBUG [Thread-1730 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8173 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:45,023 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:45,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/86584502c94d4859af1d5db4d13e630f 2024-11-08T02:32:45,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ba552a832ba04c1584448990264ed2e4 is 50, key is test_row_0/C:col10/1731033164226/Put/seqid=0 2024-11-08T02:32:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742239_1415 (size=12151) 2024-11-08T02:32:45,175 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,327 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-08T02:32:45,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033225386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:45,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033225386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033225391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033225392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,479 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:45,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ba552a832ba04c1584448990264ed2e4 2024-11-08T02:32:45,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c0ad0a223a8649309f7964265650bb84 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c0ad0a223a8649309f7964265650bb84 2024-11-08T02:32:45,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c0ad0a223a8649309f7964265650bb84, entries=200, sequenceid=167, filesize=14.2 K 2024-11-08T02:32:45,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/86584502c94d4859af1d5db4d13e630f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/86584502c94d4859af1d5db4d13e630f 2024-11-08T02:32:45,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/86584502c94d4859af1d5db4d13e630f, entries=150, sequenceid=167, filesize=11.9 K 2024-11-08T02:32:45,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ba552a832ba04c1584448990264ed2e4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ba552a832ba04c1584448990264ed2e4 2024-11-08T02:32:45,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ba552a832ba04c1584448990264ed2e4, entries=150, sequenceid=167, filesize=11.9 K 2024-11-08T02:32:45,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b16fc799559c1038c4d1f04e11c99eb6 in 1308ms, sequenceid=167, compaction requested=true 2024-11-08T02:32:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:45,536 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:45,536 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:45,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:45,537 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:45,537 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:45,537 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:45,537 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:45,537 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,537 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,537 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/20c441f326a04bb7a76af34180e9f0c6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/13a976792d6c4eb8839c93a83e2c344d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/86584502c94d4859af1d5db4d13e630f] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=35.9 K 2024-11-08T02:32:45,537 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0db2c27c976940c8821148256d694db1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c9118afefa044d75b44ebf7127a7c4e6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c0ad0a223a8649309f7964265650bb84] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=38.2 K 2024-11-08T02:32:45,538 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 20c441f326a04bb7a76af34180e9f0c6, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731033161955 2024-11-08T02:32:45,538 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0db2c27c976940c8821148256d694db1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731033161955 2024-11-08T02:32:45,538 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 13a976792d6c4eb8839c93a83e2c344d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731033162082 2024-11-08T02:32:45,538 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting c9118afefa044d75b44ebf7127a7c4e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731033162082 2024-11-08T02:32:45,538 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 86584502c94d4859af1d5db4d13e630f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731033164222 2024-11-08T02:32:45,538 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting c0ad0a223a8649309f7964265650bb84, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731033164222 2024-11-08T02:32:45,543 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:45,544 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/b272771532414eaaa3b8663c555f7b04 is 50, key is test_row_0/B:col10/1731033164226/Put/seqid=0 2024-11-08T02:32:45,546 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#349 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:45,547 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/443f7d5ac9364bb1bf2067d063e47103 is 50, key is test_row_0/A:col10/1731033164226/Put/seqid=0 2024-11-08T02:32:45,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742241_1417 (size=12561) 2024-11-08T02:32:45,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742240_1416 (size=12561) 2024-11-08T02:32:45,632 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:45,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-08T02:32:45,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,632 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:32:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:45,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:45,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4c0d4b49fabd4b3383646b9e65e9009e is 50, key is test_row_0/A:col10/1731033164266/Put/seqid=0 2024-11-08T02:32:45,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742242_1418 (size=12151) 2024-11-08T02:32:45,959 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/443f7d5ac9364bb1bf2067d063e47103 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/443f7d5ac9364bb1bf2067d063e47103 2024-11-08T02:32:45,959 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/b272771532414eaaa3b8663c555f7b04 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b272771532414eaaa3b8663c555f7b04 2024-11-08T02:32:45,963 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into 443f7d5ac9364bb1bf2067d063e47103(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:45,963 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into b272771532414eaaa3b8663c555f7b04(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:45,963 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=13, startTime=1731033165536; duration=0sec 2024-11-08T02:32:45,963 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=13, startTime=1731033165536; duration=0sec 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:45,963 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:45,964 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:45,964 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:45,964 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/7c061783bd0847f48a94dc8c45afd696, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d1892fdf2a54469b965d0c61ef2a9ad0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ba552a832ba04c1584448990264ed2e4] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=35.9 K 2024-11-08T02:32:45,964 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7c061783bd0847f48a94dc8c45afd696, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731033161955 2024-11-08T02:32:45,964 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d1892fdf2a54469b965d0c61ef2a9ad0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731033162082 2024-11-08T02:32:45,964 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ba552a832ba04c1584448990264ed2e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731033164222 2024-11-08T02:32:45,970 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:45,971 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/12644c161bcf4839ae5e213b5b394c2b is 50, key is test_row_0/C:col10/1731033164226/Put/seqid=0 2024-11-08T02:32:45,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742243_1419 (size=12561) 2024-11-08T02:32:45,980 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/12644c161bcf4839ae5e213b5b394c2b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/12644c161bcf4839ae5e213b5b394c2b 2024-11-08T02:32:45,983 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into 12644c161bcf4839ae5e213b5b394c2b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:45,983 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:45,983 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=13, startTime=1731033165536; duration=0sec 2024-11-08T02:32:45,984 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:45,984 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:46,041 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4c0d4b49fabd4b3383646b9e65e9009e 2024-11-08T02:32:46,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/ad77bba8be174468b77a6d1d53080e52 is 50, key is test_row_0/B:col10/1731033164266/Put/seqid=0 2024-11-08T02:32:46,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742244_1420 (size=12151) 2024-11-08T02:32:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-08T02:32:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033226403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033226404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033226405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033226407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,450 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/ad77bba8be174468b77a6d1d53080e52 2024-11-08T02:32:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/f874d6f6dcf74b1f97cbcaa2157d27c3 is 50, key is test_row_0/C:col10/1731033164266/Put/seqid=0 2024-11-08T02:32:46,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742245_1421 (size=12151) 2024-11-08T02:32:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033226508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033226509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033226510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033226712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033226713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033226713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:46,864 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/f874d6f6dcf74b1f97cbcaa2157d27c3 2024-11-08T02:32:46,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4c0d4b49fabd4b3383646b9e65e9009e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4c0d4b49fabd4b3383646b9e65e9009e 2024-11-08T02:32:46,874 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4c0d4b49fabd4b3383646b9e65e9009e, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T02:32:46,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/ad77bba8be174468b77a6d1d53080e52 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ad77bba8be174468b77a6d1d53080e52 2024-11-08T02:32:46,880 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ad77bba8be174468b77a6d1d53080e52, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T02:32:46,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/f874d6f6dcf74b1f97cbcaa2157d27c3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/f874d6f6dcf74b1f97cbcaa2157d27c3 2024-11-08T02:32:46,884 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/f874d6f6dcf74b1f97cbcaa2157d27c3, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T02:32:46,886 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b16fc799559c1038c4d1f04e11c99eb6 in 1254ms, sequenceid=192, compaction requested=false 2024-11-08T02:32:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:46,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-11-08T02:32:46,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=129 2024-11-08T02:32:46,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=129, resume processing ppid=128 2024-11-08T02:32:46,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=129, ppid=128, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-11-08T02:32:46,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees in 2.6270 sec 2024-11-08T02:32:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:47,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:32:47,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:47,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:47,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/306e866f3e3942a6bcaac9d7ff28e045 is 50, key is test_row_0/A:col10/1731033166395/Put/seqid=0 2024-11-08T02:32:47,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742246_1422 (size=16931) 2024-11-08T02:32:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033227055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033227055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033227056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033227164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033227165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033227165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033227368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033227369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033227369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/306e866f3e3942a6bcaac9d7ff28e045 2024-11-08T02:32:47,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/bb7aab2ca92e43259dabf5181046f625 is 50, key is test_row_0/B:col10/1731033166395/Put/seqid=0 2024-11-08T02:32:47,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742247_1423 (size=12151) 2024-11-08T02:32:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033227673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033227674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033227675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:47,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/bb7aab2ca92e43259dabf5181046f625 2024-11-08T02:32:47,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/0dc381c2138c49f2bbbed28a06eebce7 is 50, key is test_row_0/C:col10/1731033166395/Put/seqid=0 2024-11-08T02:32:47,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742248_1424 (size=12151) 2024-11-08T02:32:48,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033228176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033228182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033228184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/0dc381c2138c49f2bbbed28a06eebce7 2024-11-08T02:32:48,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/306e866f3e3942a6bcaac9d7ff28e045 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/306e866f3e3942a6bcaac9d7ff28e045 2024-11-08T02:32:48,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/306e866f3e3942a6bcaac9d7ff28e045, entries=250, sequenceid=208, filesize=16.5 K 2024-11-08T02:32:48,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/bb7aab2ca92e43259dabf5181046f625 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/bb7aab2ca92e43259dabf5181046f625 2024-11-08T02:32:48,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/bb7aab2ca92e43259dabf5181046f625, entries=150, sequenceid=208, filesize=11.9 K 2024-11-08T02:32:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/0dc381c2138c49f2bbbed28a06eebce7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0dc381c2138c49f2bbbed28a06eebce7 2024-11-08T02:32:48,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0dc381c2138c49f2bbbed28a06eebce7, entries=150, sequenceid=208, filesize=11.9 K 2024-11-08T02:32:48,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b16fc799559c1038c4d1f04e11c99eb6 in 1240ms, sequenceid=208, compaction requested=true 2024-11-08T02:32:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:48,260 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:48,260 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:48,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:48,261 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:48,261 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:48,261 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:48,261 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:48,261 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,261 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,261 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/443f7d5ac9364bb1bf2067d063e47103, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4c0d4b49fabd4b3383646b9e65e9009e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/306e866f3e3942a6bcaac9d7ff28e045] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=40.7 K 2024-11-08T02:32:48,261 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b272771532414eaaa3b8663c555f7b04, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ad77bba8be174468b77a6d1d53080e52, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/bb7aab2ca92e43259dabf5181046f625] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=36.0 K 2024-11-08T02:32:48,262 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b272771532414eaaa3b8663c555f7b04, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731033164222 2024-11-08T02:32:48,262 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 443f7d5ac9364bb1bf2067d063e47103, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731033164222 2024-11-08T02:32:48,262 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4c0d4b49fabd4b3383646b9e65e9009e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731033164259 2024-11-08T02:32:48,262 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting ad77bba8be174468b77a6d1d53080e52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731033164259 2024-11-08T02:32:48,262 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 306e866f3e3942a6bcaac9d7ff28e045, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731033166395 2024-11-08T02:32:48,262 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting bb7aab2ca92e43259dabf5181046f625, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731033166395 2024-11-08T02:32:48,267 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:48,267 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8667f420f3a2490e96f2d8c752cbff19 is 50, key is test_row_0/B:col10/1731033166395/Put/seqid=0 2024-11-08T02:32:48,277 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:48,277 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/e11ebaddfdda484796e63d4c767befcf is 50, key is test_row_0/A:col10/1731033166395/Put/seqid=0 2024-11-08T02:32:48,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742249_1425 (size=12663) 2024-11-08T02:32:48,282 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8667f420f3a2490e96f2d8c752cbff19 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8667f420f3a2490e96f2d8c752cbff19 2024-11-08T02:32:48,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742250_1426 (size=12663) 2024-11-08T02:32:48,286 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into 8667f420f3a2490e96f2d8c752cbff19(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:48,286 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:48,286 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=13, startTime=1731033168260; duration=0sec 2024-11-08T02:32:48,286 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:48,286 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:48,286 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:48,287 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:48,287 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:48,287 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,287 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/12644c161bcf4839ae5e213b5b394c2b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/f874d6f6dcf74b1f97cbcaa2157d27c3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0dc381c2138c49f2bbbed28a06eebce7] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=36.0 K 2024-11-08T02:32:48,287 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 12644c161bcf4839ae5e213b5b394c2b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731033164222 2024-11-08T02:32:48,288 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f874d6f6dcf74b1f97cbcaa2157d27c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731033164259 2024-11-08T02:32:48,288 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 0dc381c2138c49f2bbbed28a06eebce7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731033166395 2024-11-08T02:32:48,292 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:48,293 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/248ca8677e6a44ca898ebe38615dc68c is 50, key is test_row_0/C:col10/1731033166395/Put/seqid=0 2024-11-08T02:32:48,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742251_1427 (size=12663) 2024-11-08T02:32:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-08T02:32:48,367 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 128 completed 2024-11-08T02:32:48,368 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees 2024-11-08T02:32:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-08T02:32:48,369 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:48,370 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:48,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:48,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T02:32:48,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:48,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:48,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:48,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:48,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:48,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:48,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4a856643459743c6b14db12c7c16aca2 is 50, key is test_row_0/A:col10/1731033167055/Put/seqid=0 2024-11-08T02:32:48,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742252_1428 (size=14541) 2024-11-08T02:32:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033228456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-08T02:32:48,521 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033228560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-08T02:32:48,673 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,685 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/e11ebaddfdda484796e63d4c767befcf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/e11ebaddfdda484796e63d4c767befcf 2024-11-08T02:32:48,689 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into e11ebaddfdda484796e63d4c767befcf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:48,689 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:48,689 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=13, startTime=1731033168260; duration=0sec 2024-11-08T02:32:48,689 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:48,689 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:48,699 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/248ca8677e6a44ca898ebe38615dc68c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/248ca8677e6a44ca898ebe38615dc68c 2024-11-08T02:32:48,702 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into 248ca8677e6a44ca898ebe38615dc68c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:48,702 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:48,702 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=13, startTime=1731033168260; duration=0sec 2024-11-08T02:32:48,702 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:48,702 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:48,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033228765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,826 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4a856643459743c6b14db12c7c16aca2 2024-11-08T02:32:48,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/a42ca26e073c47b09a2edae9b37a955a is 50, key is test_row_0/B:col10/1731033167055/Put/seqid=0 2024-11-08T02:32:48,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742253_1429 (size=12151) 2024-11-08T02:32:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-08T02:32:48,978 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:48,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:48,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033229071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,130 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033229184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033229196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033229196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/a42ca26e073c47b09a2edae9b37a955a 2024-11-08T02:32:49,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ca1d7dd9f8884853ac1b2c394fec8098 is 50, key is test_row_0/C:col10/1731033167055/Put/seqid=0 2024-11-08T02:32:49,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742254_1430 (size=12151) 2024-11-08T02:32:49,282 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:49,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:49,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,434 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-08T02:32:49,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033229574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,587 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:49,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ca1d7dd9f8884853ac1b2c394fec8098 2024-11-08T02:32:49,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4a856643459743c6b14db12c7c16aca2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4a856643459743c6b14db12c7c16aca2 2024-11-08T02:32:49,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4a856643459743c6b14db12c7c16aca2, entries=200, sequenceid=233, filesize=14.2 K 2024-11-08T02:32:49,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/a42ca26e073c47b09a2edae9b37a955a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a42ca26e073c47b09a2edae9b37a955a 2024-11-08T02:32:49,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a42ca26e073c47b09a2edae9b37a955a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-08T02:32:49,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ca1d7dd9f8884853ac1b2c394fec8098 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ca1d7dd9f8884853ac1b2c394fec8098 2024-11-08T02:32:49,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ca1d7dd9f8884853ac1b2c394fec8098, entries=150, sequenceid=233, filesize=11.9 K 2024-11-08T02:32:49,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b16fc799559c1038c4d1f04e11c99eb6 in 1240ms, sequenceid=233, compaction requested=false 2024-11-08T02:32:49,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:49,739 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:49,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-08T02:32:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:49,740 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T02:32:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:49,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/b24c28b297994b4dadb45bb314549325 is 50, key is test_row_1/A:col10/1731033168455/Put/seqid=0 2024-11-08T02:32:49,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742255_1431 (size=9757) 2024-11-08T02:32:50,148 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/b24c28b297994b4dadb45bb314549325 2024-11-08T02:32:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8a22e5ebc1de41fd9da16d916630f84f is 50, key is test_row_1/B:col10/1731033168455/Put/seqid=0 2024-11-08T02:32:50,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742256_1432 (size=9757) 2024-11-08T02:32:50,209 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8a22e5ebc1de41fd9da16d916630f84f 2024-11-08T02:32:50,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/b9a65fa7c1fd4cf7b21e64921d941910 is 50, key is test_row_1/C:col10/1731033168455/Put/seqid=0 2024-11-08T02:32:50,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742257_1433 (size=9757) 2024-11-08T02:32:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-08T02:32:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:50,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:50,618 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/b9a65fa7c1fd4cf7b21e64921d941910 2024-11-08T02:32:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/b24c28b297994b4dadb45bb314549325 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/b24c28b297994b4dadb45bb314549325 2024-11-08T02:32:50,623 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/b24c28b297994b4dadb45bb314549325, entries=100, sequenceid=247, filesize=9.5 K 2024-11-08T02:32:50,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8a22e5ebc1de41fd9da16d916630f84f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8a22e5ebc1de41fd9da16d916630f84f 2024-11-08T02:32:50,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8a22e5ebc1de41fd9da16d916630f84f, entries=100, sequenceid=247, filesize=9.5 K 2024-11-08T02:32:50,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/b9a65fa7c1fd4cf7b21e64921d941910 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/b9a65fa7c1fd4cf7b21e64921d941910 2024-11-08T02:32:50,630 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/b9a65fa7c1fd4cf7b21e64921d941910, entries=100, sequenceid=247, filesize=9.5 K 2024-11-08T02:32:50,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=93.93 KB/96180 for b16fc799559c1038c4d1f04e11c99eb6 in 904ms, sequenceid=247, compaction requested=true 2024-11-08T02:32:50,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:50,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:50,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-11-08T02:32:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:50,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:32:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-11-08T02:32:50,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:50,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:50,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:50,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:50,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:50,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:50,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-11-08T02:32:50,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2750 sec 2024-11-08T02:32:50,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees in 2.2790 sec 2024-11-08T02:32:50,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/3bd8212bb3d0458f80256514b8e31a03 is 50, key is test_row_0/A:col10/1731033170629/Put/seqid=0 2024-11-08T02:32:50,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742258_1434 (size=14741) 2024-11-08T02:32:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033230722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:50,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033230830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:51,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033231034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:51,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/3bd8212bb3d0458f80256514b8e31a03 2024-11-08T02:32:51,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/847b7fb6e9344097afc7c5a629453b14 is 50, key is test_row_0/B:col10/1731033170629/Put/seqid=0 2024-11-08T02:32:51,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742259_1435 (size=12301) 2024-11-08T02:32:51,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033231199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:51,203 DEBUG [Thread-1728 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4149 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:51,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033231203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:51,207 DEBUG [Thread-1732 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4151 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:51,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033231221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:51,226 DEBUG [Thread-1726 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4171 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:51,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033231338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:51,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/847b7fb6e9344097afc7c5a629453b14 2024-11-08T02:32:51,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/6f40a3f4c4c840f5b7b56b724822728a is 50, key is test_row_0/C:col10/1731033170629/Put/seqid=0 2024-11-08T02:32:51,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742260_1436 (size=12301) 2024-11-08T02:32:51,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033231844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:51,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/6f40a3f4c4c840f5b7b56b724822728a 2024-11-08T02:32:51,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/3bd8212bb3d0458f80256514b8e31a03 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/3bd8212bb3d0458f80256514b8e31a03 2024-11-08T02:32:51,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/3bd8212bb3d0458f80256514b8e31a03, entries=200, sequenceid=265, filesize=14.4 K 2024-11-08T02:32:51,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/847b7fb6e9344097afc7c5a629453b14 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/847b7fb6e9344097afc7c5a629453b14 2024-11-08T02:32:51,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/847b7fb6e9344097afc7c5a629453b14, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T02:32:51,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/6f40a3f4c4c840f5b7b56b724822728a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/6f40a3f4c4c840f5b7b56b724822728a 2024-11-08T02:32:51,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/6f40a3f4c4c840f5b7b56b724822728a, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T02:32:51,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b16fc799559c1038c4d1f04e11c99eb6 in 1246ms, sequenceid=265, compaction requested=true 2024-11-08T02:32:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:51,890 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:32:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:51,890 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:32:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:51,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:51,891 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:32:51,891 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51702 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:32:51,891 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:51,891 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:51,891 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:51,891 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:51,891 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8667f420f3a2490e96f2d8c752cbff19, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a42ca26e073c47b09a2edae9b37a955a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8a22e5ebc1de41fd9da16d916630f84f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/847b7fb6e9344097afc7c5a629453b14] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=45.8 K 2024-11-08T02:32:51,891 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/e11ebaddfdda484796e63d4c767befcf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4a856643459743c6b14db12c7c16aca2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/b24c28b297994b4dadb45bb314549325, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/3bd8212bb3d0458f80256514b8e31a03] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=50.5 K 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 8667f420f3a2490e96f2d8c752cbff19, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731033166395 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting e11ebaddfdda484796e63d4c767befcf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731033166395 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4a856643459743c6b14db12c7c16aca2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033167044 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a42ca26e073c47b09a2edae9b37a955a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033167052 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b24c28b297994b4dadb45bb314549325, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731033168445 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 8a22e5ebc1de41fd9da16d916630f84f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731033168445 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3bd8212bb3d0458f80256514b8e31a03, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731033170611 2024-11-08T02:32:51,892 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 847b7fb6e9344097afc7c5a629453b14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731033170611 2024-11-08T02:32:51,900 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:51,900 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:51,901 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c3e26b2dd4fe4364a7c269409f11cfb7 is 50, key is test_row_0/A:col10/1731033170629/Put/seqid=0 2024-11-08T02:32:51,901 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/991b533ad0e342f799d16f978ea2c21d is 50, key is test_row_0/B:col10/1731033170629/Put/seqid=0 2024-11-08T02:32:51,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742261_1437 (size=12949) 2024-11-08T02:32:51,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742262_1438 (size=12949) 2024-11-08T02:32:51,920 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/c3e26b2dd4fe4364a7c269409f11cfb7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c3e26b2dd4fe4364a7c269409f11cfb7 2024-11-08T02:32:51,925 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into c3e26b2dd4fe4364a7c269409f11cfb7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:51,925 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:51,925 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=12, startTime=1731033171890; duration=0sec 2024-11-08T02:32:51,925 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:51,925 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:51,925 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:32:51,926 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46872 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:32:51,926 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:51,926 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:51,926 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/248ca8677e6a44ca898ebe38615dc68c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ca1d7dd9f8884853ac1b2c394fec8098, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/b9a65fa7c1fd4cf7b21e64921d941910, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/6f40a3f4c4c840f5b7b56b724822728a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=45.8 K 2024-11-08T02:32:51,927 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 248ca8677e6a44ca898ebe38615dc68c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731033166395 2024-11-08T02:32:51,927 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ca1d7dd9f8884853ac1b2c394fec8098, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731033167052 2024-11-08T02:32:51,927 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b9a65fa7c1fd4cf7b21e64921d941910, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731033168445 2024-11-08T02:32:51,927 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6f40a3f4c4c840f5b7b56b724822728a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731033170611 2024-11-08T02:32:51,933 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:51,933 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/3ba58a34152c4c098a86a7fb51ff251a is 50, key is test_row_0/C:col10/1731033170629/Put/seqid=0 2024-11-08T02:32:51,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742263_1439 (size=12949) 2024-11-08T02:32:52,319 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/991b533ad0e342f799d16f978ea2c21d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/991b533ad0e342f799d16f978ea2c21d 2024-11-08T02:32:52,322 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into 991b533ad0e342f799d16f978ea2c21d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:52,322 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:52,322 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=12, startTime=1731033171890; duration=0sec 2024-11-08T02:32:52,322 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:52,322 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:52,346 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/3ba58a34152c4c098a86a7fb51ff251a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3ba58a34152c4c098a86a7fb51ff251a 2024-11-08T02:32:52,350 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into 3ba58a34152c4c098a86a7fb51ff251a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:52,350 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:52,350 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=12, startTime=1731033171890; duration=0sec 2024-11-08T02:32:52,350 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:52,350 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-08T02:32:52,473 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 130 completed 2024-11-08T02:32:52,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-11-08T02:32:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-08T02:32:52,475 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:52,476 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:52,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:52,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-08T02:32:52,627 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:52,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-08T02:32:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:52,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T02:32:52,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:52,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:52,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/06dbb542278643db8b5dce632414143d is 50, key is test_row_0/A:col10/1731033170707/Put/seqid=0 2024-11-08T02:32:52,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742264_1440 (size=12301) 2024-11-08T02:32:52,634 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/06dbb542278643db8b5dce632414143d 2024-11-08T02:32:52,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/4b36c0197c864ae8beb7a026a8da9810 is 50, key is test_row_0/B:col10/1731033170707/Put/seqid=0 2024-11-08T02:32:52,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742265_1441 (size=12301) 2024-11-08T02:32:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-08T02:32:52,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:52,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:52,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:52,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033232907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:53,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033233011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:53,043 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/4b36c0197c864ae8beb7a026a8da9810 2024-11-08T02:32:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/97b81da9998b4091a17e814e5dbdbd38 is 50, key is test_row_0/C:col10/1731033170707/Put/seqid=0 2024-11-08T02:32:53,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742266_1442 (size=12301) 2024-11-08T02:32:53,058 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/97b81da9998b4091a17e814e5dbdbd38 2024-11-08T02:32:53,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/06dbb542278643db8b5dce632414143d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/06dbb542278643db8b5dce632414143d 2024-11-08T02:32:53,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/06dbb542278643db8b5dce632414143d, entries=150, sequenceid=286, filesize=12.0 K 2024-11-08T02:32:53,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/4b36c0197c864ae8beb7a026a8da9810 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4b36c0197c864ae8beb7a026a8da9810 2024-11-08T02:32:53,071 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4b36c0197c864ae8beb7a026a8da9810, entries=150, sequenceid=286, filesize=12.0 K 2024-11-08T02:32:53,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/97b81da9998b4091a17e814e5dbdbd38 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/97b81da9998b4091a17e814e5dbdbd38 2024-11-08T02:32:53,075 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/97b81da9998b4091a17e814e5dbdbd38, entries=150, sequenceid=286, filesize=12.0 K 2024-11-08T02:32:53,076 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b16fc799559c1038c4d1f04e11c99eb6 in 448ms, sequenceid=286, compaction requested=false 2024-11-08T02:32:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:53,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-11-08T02:32:53,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=133 2024-11-08T02:32:53,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-08T02:32:53,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=133, resume processing ppid=132 2024-11-08T02:32:53,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, ppid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 600 msec 2024-11-08T02:32:53,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 604 msec 2024-11-08T02:32:53,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:53,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T02:32:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:53,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:53,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/ea3963cb29f742848b19f6d397c7aba2 is 50, key is test_row_0/A:col10/1731033172903/Put/seqid=0 2024-11-08T02:32:53,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742267_1443 (size=12301) 2024-11-08T02:32:53,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033233276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033233382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-08T02:32:53,577 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 132 completed 2024-11-08T02:32:53,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees 2024-11-08T02:32:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-08T02:32:53,579 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:53,580 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=134, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:53,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033233587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:53,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/ea3963cb29f742848b19f6d397c7aba2 2024-11-08T02:32:53,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/205da49022ac4652a7dc35471e768f03 is 50, key is test_row_0/B:col10/1731033172903/Put/seqid=0 2024-11-08T02:32:53,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742268_1444 (size=12301) 2024-11-08T02:32:53,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/205da49022ac4652a7dc35471e768f03 2024-11-08T02:32:53,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/792724f421bc4c8ea3da85e0166f53c7 is 50, key is test_row_0/C:col10/1731033172903/Put/seqid=0 2024-11-08T02:32:53,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742269_1445 (size=12301) 2024-11-08T02:32:53,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/792724f421bc4c8ea3da85e0166f53c7 2024-11-08T02:32:53,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/ea3963cb29f742848b19f6d397c7aba2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ea3963cb29f742848b19f6d397c7aba2 2024-11-08T02:32:53,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ea3963cb29f742848b19f6d397c7aba2, entries=150, sequenceid=305, filesize=12.0 K 2024-11-08T02:32:53,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/205da49022ac4652a7dc35471e768f03 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/205da49022ac4652a7dc35471e768f03 2024-11-08T02:32:53,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/205da49022ac4652a7dc35471e768f03, entries=150, sequenceid=305, filesize=12.0 K 2024-11-08T02:32:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/792724f421bc4c8ea3da85e0166f53c7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/792724f421bc4c8ea3da85e0166f53c7 2024-11-08T02:32:53,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/792724f421bc4c8ea3da85e0166f53c7, entries=150, sequenceid=305, filesize=12.0 K 2024-11-08T02:32:53,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b16fc799559c1038c4d1f04e11c99eb6 in 451ms, sequenceid=305, compaction requested=true 2024-11-08T02:32:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:53,669 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:53,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:53,669 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:53,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:53,671 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:53,671 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:53,671 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:53,671 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:53,671 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:53,671 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:53,671 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c3e26b2dd4fe4364a7c269409f11cfb7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/06dbb542278643db8b5dce632414143d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ea3963cb29f742848b19f6d397c7aba2] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=36.7 K 2024-11-08T02:32:53,671 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/991b533ad0e342f799d16f978ea2c21d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4b36c0197c864ae8beb7a026a8da9810, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/205da49022ac4652a7dc35471e768f03] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=36.7 K 2024-11-08T02:32:53,671 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting c3e26b2dd4fe4364a7c269409f11cfb7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731033170611 2024-11-08T02:32:53,671 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 991b533ad0e342f799d16f978ea2c21d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731033170611 2024-11-08T02:32:53,672 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 4b36c0197c864ae8beb7a026a8da9810, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731033170696 2024-11-08T02:32:53,672 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 06dbb542278643db8b5dce632414143d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731033170696 2024-11-08T02:32:53,672 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 205da49022ac4652a7dc35471e768f03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731033172903 2024-11-08T02:32:53,672 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting ea3963cb29f742848b19f6d397c7aba2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731033172903 2024-11-08T02:32:53,680 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:53,680 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:53,681 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/f758b866aaec47418c5cdd6b4bd65a10 is 50, key is test_row_0/B:col10/1731033172903/Put/seqid=0 2024-11-08T02:32:53,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-08T02:32:53,682 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/084a617d4435474696a407d2879ef8e4 is 50, key is test_row_0/A:col10/1731033172903/Put/seqid=0 2024-11-08T02:32:53,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742270_1446 (size=13051) 2024-11-08T02:32:53,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742271_1447 (size=13051) 2024-11-08T02:32:53,731 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:53,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=135 2024-11-08T02:32:53,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:53,731 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:32:53,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:53,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:53,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:53,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:53,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:53,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:53,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/8d2579fe10a24c7c8226e67e6276c059 is 50, key is test_row_0/A:col10/1731033173268/Put/seqid=0 2024-11-08T02:32:53,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742272_1448 (size=12301) 2024-11-08T02:32:53,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-08T02:32:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:53,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:53,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:53,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033233964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:54,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:54,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033234066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:54,095 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/f758b866aaec47418c5cdd6b4bd65a10 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/f758b866aaec47418c5cdd6b4bd65a10 2024-11-08T02:32:54,095 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/084a617d4435474696a407d2879ef8e4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/084a617d4435474696a407d2879ef8e4 2024-11-08T02:32:54,101 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into f758b866aaec47418c5cdd6b4bd65a10(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:54,101 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into 084a617d4435474696a407d2879ef8e4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:54,101 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:54,101 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:54,101 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=13, startTime=1731033173669; duration=0sec 2024-11-08T02:32:54,101 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=13, startTime=1731033173669; duration=0sec 2024-11-08T02:32:54,101 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:54,101 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:54,101 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:54,101 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:54,101 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:54,102 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:54,102 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:54,102 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:54,102 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3ba58a34152c4c098a86a7fb51ff251a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/97b81da9998b4091a17e814e5dbdbd38, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/792724f421bc4c8ea3da85e0166f53c7] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=36.7 K 2024-11-08T02:32:54,103 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3ba58a34152c4c098a86a7fb51ff251a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731033170611 2024-11-08T02:32:54,103 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 97b81da9998b4091a17e814e5dbdbd38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731033170696 2024-11-08T02:32:54,103 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 792724f421bc4c8ea3da85e0166f53c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731033172903 2024-11-08T02:32:54,108 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:54,109 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/d0c8fe1d20f54b7ebcc71aa8d218ac0e is 50, key is test_row_0/C:col10/1731033172903/Put/seqid=0 2024-11-08T02:32:54,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742273_1449 (size=13051) 2024-11-08T02:32:54,138 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/8d2579fe10a24c7c8226e67e6276c059 2024-11-08T02:32:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6ab64263c4844fc594af0111cd86e6f2 is 50, key is test_row_0/B:col10/1731033173268/Put/seqid=0 2024-11-08T02:32:54,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742274_1450 (size=12301) 2024-11-08T02:32:54,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-08T02:32:54,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033234273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:54,515 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/d0c8fe1d20f54b7ebcc71aa8d218ac0e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d0c8fe1d20f54b7ebcc71aa8d218ac0e 2024-11-08T02:32:54,518 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into d0c8fe1d20f54b7ebcc71aa8d218ac0e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:54,518 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:54,518 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=13, startTime=1731033173670; duration=0sec 2024-11-08T02:32:54,518 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:54,518 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:54,548 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6ab64263c4844fc594af0111cd86e6f2 2024-11-08T02:32:54,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ffe2cc8bcd8b4cce970fa5802099c4dc is 50, key is test_row_0/C:col10/1731033173268/Put/seqid=0 2024-11-08T02:32:54,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742275_1451 (size=12301) 2024-11-08T02:32:54,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033234578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-08T02:32:54,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ffe2cc8bcd8b4cce970fa5802099c4dc 2024-11-08T02:32:54,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/8d2579fe10a24c7c8226e67e6276c059 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/8d2579fe10a24c7c8226e67e6276c059 2024-11-08T02:32:54,970 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/8d2579fe10a24c7c8226e67e6276c059, entries=150, sequenceid=322, filesize=12.0 K 2024-11-08T02:32:54,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/6ab64263c4844fc594af0111cd86e6f2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6ab64263c4844fc594af0111cd86e6f2 2024-11-08T02:32:54,973 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6ab64263c4844fc594af0111cd86e6f2, entries=150, sequenceid=322, filesize=12.0 K 2024-11-08T02:32:54,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/ffe2cc8bcd8b4cce970fa5802099c4dc as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ffe2cc8bcd8b4cce970fa5802099c4dc 2024-11-08T02:32:54,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ffe2cc8bcd8b4cce970fa5802099c4dc, entries=150, sequenceid=322, filesize=12.0 K 2024-11-08T02:32:54,978 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b16fc799559c1038c4d1f04e11c99eb6 in 1247ms, sequenceid=322, compaction requested=false 2024-11-08T02:32:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=135}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=135 2024-11-08T02:32:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=135 2024-11-08T02:32:54,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=135, resume processing ppid=134 2024-11-08T02:32:54,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, ppid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-11-08T02:32:54,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=134, table=TestAcidGuarantees in 1.4020 sec 2024-11-08T02:32:55,057 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:32:55,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:55,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T02:32:55,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:55,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:55,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:55,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:55,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:55,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:55,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0cade48a41ec4d7abc76d0e73924533b is 50, key is test_row_0/A:col10/1731033175083/Put/seqid=0 2024-11-08T02:32:55,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742276_1452 (size=14741) 2024-11-08T02:32:55,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033235130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033235130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37504 deadline: 1731033235207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,216 DEBUG [Thread-1728 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8161 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:55,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37560 deadline: 1731033235224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,227 DEBUG [Thread-1732 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8170 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:55,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033235233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033235233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1731033235245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,250 DEBUG [Thread-1726 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8195 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., hostname=ecdbff65922a,40577,1731033026611, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:32:55,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033235437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033235438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0cade48a41ec4d7abc76d0e73924533b 2024-11-08T02:32:55,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8dd71c1bcffc42529bb84ae761e5bd94 is 50, key is test_row_0/B:col10/1731033175083/Put/seqid=0 2024-11-08T02:32:55,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742277_1453 (size=12301) 2024-11-08T02:32:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-08T02:32:55,684 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 134 completed 2024-11-08T02:32:55,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:32:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-11-08T02:32:55,686 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:32:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-11-08T02:32:55,687 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:32:55,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:32:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033235741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033235742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-11-08T02:32:55,838 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-08T02:32:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:55,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:55,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8dd71c1bcffc42529bb84ae761e5bd94 2024-11-08T02:32:55,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/fc1c7765592144b9ad0a4dab767d113a is 50, key is test_row_0/C:col10/1731033175083/Put/seqid=0 2024-11-08T02:32:55,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742278_1454 (size=12301) 2024-11-08T02:32:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-11-08T02:32:55,990 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:55,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-08T02:32:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:55,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:55,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:56,142 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:56,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-08T02:32:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:56,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:56,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:56,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37518 deadline: 1731033236246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:56,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:32:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37582 deadline: 1731033236248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:32:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-11-08T02:32:56,294 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:56,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-08T02:32:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:56,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:32:56,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/fc1c7765592144b9ad0a4dab767d113a 2024-11-08T02:32:56,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/0cade48a41ec4d7abc76d0e73924533b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0cade48a41ec4d7abc76d0e73924533b 2024-11-08T02:32:56,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0cade48a41ec4d7abc76d0e73924533b, entries=200, sequenceid=345, filesize=14.4 K 2024-11-08T02:32:56,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/8dd71c1bcffc42529bb84ae761e5bd94 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8dd71c1bcffc42529bb84ae761e5bd94 2024-11-08T02:32:56,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8dd71c1bcffc42529bb84ae761e5bd94, entries=150, sequenceid=345, filesize=12.0 K 2024-11-08T02:32:56,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/fc1c7765592144b9ad0a4dab767d113a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/fc1c7765592144b9ad0a4dab767d113a 2024-11-08T02:32:56,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/fc1c7765592144b9ad0a4dab767d113a, entries=150, sequenceid=345, filesize=12.0 K 2024-11-08T02:32:56,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for b16fc799559c1038c4d1f04e11c99eb6 in 1291ms, sequenceid=345, compaction requested=true 2024-11-08T02:32:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:56,375 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:32:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:56,375 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:32:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b16fc799559c1038c4d1f04e11c99eb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:32:56,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:56,376 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:56,376 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/A is initiating minor compaction (all files) 2024-11-08T02:32:56,376 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/A in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,376 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/084a617d4435474696a407d2879ef8e4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/8d2579fe10a24c7c8226e67e6276c059, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0cade48a41ec4d7abc76d0e73924533b] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=39.2 K 2024-11-08T02:32:56,376 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:56,376 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/B is initiating minor compaction (all files) 2024-11-08T02:32:56,376 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/B in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,376 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/f758b866aaec47418c5cdd6b4bd65a10, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6ab64263c4844fc594af0111cd86e6f2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8dd71c1bcffc42529bb84ae761e5bd94] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=36.8 K 2024-11-08T02:32:56,377 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 084a617d4435474696a407d2879ef8e4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731033172903 2024-11-08T02:32:56,377 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f758b866aaec47418c5cdd6b4bd65a10, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731033172903 2024-11-08T02:32:56,377 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 6ab64263c4844fc594af0111cd86e6f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731033173268 2024-11-08T02:32:56,377 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8d2579fe10a24c7c8226e67e6276c059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731033173268 2024-11-08T02:32:56,377 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 8dd71c1bcffc42529bb84ae761e5bd94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731033173945 2024-11-08T02:32:56,377 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0cade48a41ec4d7abc76d0e73924533b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731033173945 2024-11-08T02:32:56,384 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#A#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:56,384 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/35e3bdf2bd9a46269c5c2ade02ed86b9 is 50, key is test_row_0/A:col10/1731033175083/Put/seqid=0 2024-11-08T02:32:56,388 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:56,389 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/4907739ce7df4517a1519e651655f16c is 50, key is test_row_0/B:col10/1731033175083/Put/seqid=0 2024-11-08T02:32:56,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742279_1455 (size=13153) 2024-11-08T02:32:56,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742280_1456 (size=13153) 2024-11-08T02:32:56,405 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/4907739ce7df4517a1519e651655f16c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4907739ce7df4517a1519e651655f16c 2024-11-08T02:32:56,408 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/B of b16fc799559c1038c4d1f04e11c99eb6 into 4907739ce7df4517a1519e651655f16c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:56,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:56,409 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/B, priority=13, startTime=1731033176375; duration=0sec 2024-11-08T02:32:56,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:32:56,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:B 2024-11-08T02:32:56,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:32:56,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:32:56,409 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): b16fc799559c1038c4d1f04e11c99eb6/C is initiating minor compaction (all files) 2024-11-08T02:32:56,409 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b16fc799559c1038c4d1f04e11c99eb6/C in TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,409 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d0c8fe1d20f54b7ebcc71aa8d218ac0e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ffe2cc8bcd8b4cce970fa5802099c4dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/fc1c7765592144b9ad0a4dab767d113a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp, totalSize=36.8 K 2024-11-08T02:32:56,410 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting d0c8fe1d20f54b7ebcc71aa8d218ac0e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1731033172903 2024-11-08T02:32:56,410 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting ffe2cc8bcd8b4cce970fa5802099c4dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731033173268 2024-11-08T02:32:56,410 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting fc1c7765592144b9ad0a4dab767d113a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731033173945 2024-11-08T02:32:56,417 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b16fc799559c1038c4d1f04e11c99eb6#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:32:56,417 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/949f52cede434c7d81991a5e4b2ff45a is 50, key is test_row_0/C:col10/1731033175083/Put/seqid=0 2024-11-08T02:32:56,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742281_1457 (size=13153) 2024-11-08T02:32:56,426 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/949f52cede434c7d81991a5e4b2ff45a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/949f52cede434c7d81991a5e4b2ff45a 2024-11-08T02:32:56,430 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/C of b16fc799559c1038c4d1f04e11c99eb6 into 949f52cede434c7d81991a5e4b2ff45a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:56,430 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:56,430 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/C, priority=13, startTime=1731033176375; duration=0sec 2024-11-08T02:32:56,430 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:56,430 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:C 2024-11-08T02:32:56,447 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:32:56,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-08T02:32:56,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:56,447 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T02:32:56,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:32:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:32:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:32:56,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:32:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4b2e153711804a94aa30bb4b65b71eeb is 50, key is test_row_0/A:col10/1731033175129/Put/seqid=0 2024-11-08T02:32:56,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742282_1458 (size=12301) 2024-11-08T02:32:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-11-08T02:32:56,805 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/35e3bdf2bd9a46269c5c2ade02ed86b9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/35e3bdf2bd9a46269c5c2ade02ed86b9 2024-11-08T02:32:56,809 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b16fc799559c1038c4d1f04e11c99eb6/A of b16fc799559c1038c4d1f04e11c99eb6 into 35e3bdf2bd9a46269c5c2ade02ed86b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:32:56,809 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:56,809 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6., storeName=b16fc799559c1038c4d1f04e11c99eb6/A, priority=13, startTime=1731033176375; duration=0sec 2024-11-08T02:32:56,810 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:32:56,810 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b16fc799559c1038c4d1f04e11c99eb6:A 2024-11-08T02:32:56,841 DEBUG [Thread-1741 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6dd48863 to 127.0.0.1:56877 2024-11-08T02:32:56,841 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:56,842 DEBUG [Thread-1739 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6050584c to 127.0.0.1:56877 2024-11-08T02:32:56,842 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:56,844 DEBUG [Thread-1743 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x51196534 to 127.0.0.1:56877 2024-11-08T02:32:56,844 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:56,845 DEBUG [Thread-1737 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2070263a to 127.0.0.1:56877 2024-11-08T02:32:56,845 DEBUG [Thread-1737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:56,847 DEBUG [Thread-1745 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1dc5e114 to 127.0.0.1:56877 2024-11-08T02:32:56,847 DEBUG [Thread-1745 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:56,856 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4b2e153711804a94aa30bb4b65b71eeb 2024-11-08T02:32:56,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/099e735053db456b88f7747954785111 is 50, key is test_row_0/B:col10/1731033175129/Put/seqid=0 2024-11-08T02:32:56,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742283_1459 (size=12301) 2024-11-08T02:32:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:32:57,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. as already flushing 2024-11-08T02:32:57,261 DEBUG [Thread-1734 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b7f20c4 to 127.0.0.1:56877 2024-11-08T02:32:57,262 DEBUG [Thread-1734 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:57,264 DEBUG [Thread-1730 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5871c039 to 127.0.0.1:56877 2024-11-08T02:32:57,264 DEBUG [Thread-1730 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:32:57,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/099e735053db456b88f7747954785111 2024-11-08T02:32:57,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/448b47b0d4b2408eb435aa0bf3e07b5b is 50, key is test_row_0/C:col10/1731033175129/Put/seqid=0 2024-11-08T02:32:57,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742284_1460 (size=12301) 2024-11-08T02:32:57,681 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/448b47b0d4b2408eb435aa0bf3e07b5b 2024-11-08T02:32:57,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/4b2e153711804a94aa30bb4b65b71eeb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4b2e153711804a94aa30bb4b65b71eeb 2024-11-08T02:32:57,695 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4b2e153711804a94aa30bb4b65b71eeb, entries=150, sequenceid=364, filesize=12.0 K 2024-11-08T02:32:57,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/099e735053db456b88f7747954785111 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/099e735053db456b88f7747954785111 2024-11-08T02:32:57,700 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/099e735053db456b88f7747954785111, entries=150, sequenceid=364, filesize=12.0 K 2024-11-08T02:32:57,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/448b47b0d4b2408eb435aa0bf3e07b5b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/448b47b0d4b2408eb435aa0bf3e07b5b 2024-11-08T02:32:57,704 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/448b47b0d4b2408eb435aa0bf3e07b5b, entries=150, sequenceid=364, filesize=12.0 K 2024-11-08T02:32:57,705 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=13.42 KB/13740 for b16fc799559c1038c4d1f04e11c99eb6 in 1257ms, sequenceid=364, compaction requested=false 2024-11-08T02:32:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2538): Flush status journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:32:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:32:57,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-11-08T02:32:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=137 2024-11-08T02:32:57,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=137, resume processing ppid=136 2024-11-08T02:32:57,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, ppid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0190 sec 2024-11-08T02:32:57,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 2.0220 sec 2024-11-08T02:32:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=136 2024-11-08T02:32:57,790 INFO [Thread-1736 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 136 completed 2024-11-08T02:33:05,251 DEBUG [Thread-1728 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7d0ab200 to 127.0.0.1:56877 2024-11-08T02:33:05,251 DEBUG [Thread-1728 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:05,307 DEBUG [Thread-1732 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7daa5922 to 127.0.0.1:56877 2024-11-08T02:33:05,307 DEBUG [Thread-1732 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:05,313 DEBUG [Thread-1726 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x65df2359 to 127.0.0.1:56877 2024-11-08T02:33:05,313 DEBUG [Thread-1726 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2453 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7357 rows 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2445 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7335 rows 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2432 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7296 rows 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2451 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7353 rows 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2441 2024-11-08T02:33:05,313 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7323 rows 2024-11-08T02:33:05,313 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T02:33:05,314 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63cefe40 to 127.0.0.1:56877 2024-11-08T02:33:05,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:05,315 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-08T02:33:05,316 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:33:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=138, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-08T02:33:05,318 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033185318"}]},"ts":"1731033185318"} 2024-11-08T02:33:05,319 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:33:05,337 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:33:05,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:33:05,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b16fc799559c1038c4d1f04e11c99eb6, UNASSIGN}] 2024-11-08T02:33:05,340 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b16fc799559c1038c4d1f04e11c99eb6, UNASSIGN 2024-11-08T02:33:05,341 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=b16fc799559c1038c4d1f04e11c99eb6, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:05,342 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:33:05,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; CloseRegionProcedure b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:33:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-08T02:33:05,494 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:05,495 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] handler.UnassignRegionHandler(124): Close b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:33:05,495 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:33:05,496 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1681): Closing b16fc799559c1038c4d1f04e11c99eb6, disabling compactions & flushes 2024-11-08T02:33:05,496 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:33:05,496 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:33:05,496 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. after waiting 0 ms 2024-11-08T02:33:05,496 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:33:05,496 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(2837): Flushing b16fc799559c1038c4d1f04e11c99eb6 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T02:33:05,497 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=A 2024-11-08T02:33:05,497 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:05,497 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=B 2024-11-08T02:33:05,497 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:05,497 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b16fc799559c1038c4d1f04e11c99eb6, store=C 2024-11-08T02:33:05,498 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:05,503 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/21b7ef90495a491e98efc08b0dd0e193 is 50, key is test_row_0/A:col10/1731033177263/Put/seqid=0 2024-11-08T02:33:05,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742285_1461 (size=12301) 2024-11-08T02:33:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-08T02:33:05,909 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/21b7ef90495a491e98efc08b0dd0e193 2024-11-08T02:33:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-08T02:33:05,922 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/7eaf5d815a994fbdaf3248b0a8ead188 is 50, key is test_row_0/B:col10/1731033177263/Put/seqid=0 2024-11-08T02:33:05,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742286_1462 (size=12301) 2024-11-08T02:33:06,328 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/7eaf5d815a994fbdaf3248b0a8ead188 2024-11-08T02:33:06,342 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/2a8455de7883468aa47f3c2200a353ec is 50, key is test_row_0/C:col10/1731033177263/Put/seqid=0 2024-11-08T02:33:06,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742287_1463 (size=12301) 2024-11-08T02:33:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-08T02:33:06,748 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/2a8455de7883468aa47f3c2200a353ec 2024-11-08T02:33:06,759 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/A/21b7ef90495a491e98efc08b0dd0e193 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/21b7ef90495a491e98efc08b0dd0e193 2024-11-08T02:33:06,763 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/21b7ef90495a491e98efc08b0dd0e193, entries=150, sequenceid=373, filesize=12.0 K 2024-11-08T02:33:06,763 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/B/7eaf5d815a994fbdaf3248b0a8ead188 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/7eaf5d815a994fbdaf3248b0a8ead188 2024-11-08T02:33:06,767 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/7eaf5d815a994fbdaf3248b0a8ead188, entries=150, sequenceid=373, filesize=12.0 K 2024-11-08T02:33:06,767 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/.tmp/C/2a8455de7883468aa47f3c2200a353ec as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/2a8455de7883468aa47f3c2200a353ec 2024-11-08T02:33:06,770 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/2a8455de7883468aa47f3c2200a353ec, entries=150, sequenceid=373, filesize=12.0 K 2024-11-08T02:33:06,771 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b16fc799559c1038c4d1f04e11c99eb6 in 1275ms, sequenceid=373, compaction requested=true 2024-11-08T02:33:06,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1e85bc2b30c54b29809509cc2e7b3db2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/493dcd911b4f4458aca70988e4936ff8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1c957260a38848349d01ec9b94a09d00, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/bd4eb8f758b14b799528069b4e616925, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/edb682603db544ac86cc2461c67f2065, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/53e6002783294dbbb05405054b9f591a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/38797736308b4f0fb2b1164e01d98e79, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0a48931e5b2a456f8f397746b69b23bc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0db2c27c976940c8821148256d694db1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c9118afefa044d75b44ebf7127a7c4e6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c0ad0a223a8649309f7964265650bb84, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/443f7d5ac9364bb1bf2067d063e47103, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4c0d4b49fabd4b3383646b9e65e9009e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/306e866f3e3942a6bcaac9d7ff28e045, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/e11ebaddfdda484796e63d4c767befcf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4a856643459743c6b14db12c7c16aca2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/b24c28b297994b4dadb45bb314549325, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/3bd8212bb3d0458f80256514b8e31a03, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c3e26b2dd4fe4364a7c269409f11cfb7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/06dbb542278643db8b5dce632414143d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/084a617d4435474696a407d2879ef8e4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ea3963cb29f742848b19f6d397c7aba2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/8d2579fe10a24c7c8226e67e6276c059, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0cade48a41ec4d7abc76d0e73924533b] to archive 2024-11-08T02:33:06,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:33:06,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1e85bc2b30c54b29809509cc2e7b3db2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1e85bc2b30c54b29809509cc2e7b3db2 2024-11-08T02:33:06,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/493dcd911b4f4458aca70988e4936ff8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/493dcd911b4f4458aca70988e4936ff8 2024-11-08T02:33:06,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1c957260a38848349d01ec9b94a09d00 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/1c957260a38848349d01ec9b94a09d00 2024-11-08T02:33:06,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/bd4eb8f758b14b799528069b4e616925 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/bd4eb8f758b14b799528069b4e616925 2024-11-08T02:33:06,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/edb682603db544ac86cc2461c67f2065 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/edb682603db544ac86cc2461c67f2065 2024-11-08T02:33:06,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/53e6002783294dbbb05405054b9f591a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/53e6002783294dbbb05405054b9f591a 2024-11-08T02:33:06,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/38797736308b4f0fb2b1164e01d98e79 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/38797736308b4f0fb2b1164e01d98e79 2024-11-08T02:33:06,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ec497c4ef2ec4fbcbcecb93bfe12bd8d 2024-11-08T02:33:06,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0a48931e5b2a456f8f397746b69b23bc to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0a48931e5b2a456f8f397746b69b23bc 2024-11-08T02:33:06,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0db2c27c976940c8821148256d694db1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0db2c27c976940c8821148256d694db1 2024-11-08T02:33:06,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c9118afefa044d75b44ebf7127a7c4e6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c9118afefa044d75b44ebf7127a7c4e6 2024-11-08T02:33:06,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c0ad0a223a8649309f7964265650bb84 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c0ad0a223a8649309f7964265650bb84 2024-11-08T02:33:06,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/443f7d5ac9364bb1bf2067d063e47103 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/443f7d5ac9364bb1bf2067d063e47103 2024-11-08T02:33:06,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4c0d4b49fabd4b3383646b9e65e9009e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4c0d4b49fabd4b3383646b9e65e9009e 2024-11-08T02:33:06,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/306e866f3e3942a6bcaac9d7ff28e045 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/306e866f3e3942a6bcaac9d7ff28e045 2024-11-08T02:33:06,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/e11ebaddfdda484796e63d4c767befcf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/e11ebaddfdda484796e63d4c767befcf 2024-11-08T02:33:06,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4a856643459743c6b14db12c7c16aca2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4a856643459743c6b14db12c7c16aca2 2024-11-08T02:33:06,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/b24c28b297994b4dadb45bb314549325 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/b24c28b297994b4dadb45bb314549325 2024-11-08T02:33:06,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/3bd8212bb3d0458f80256514b8e31a03 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/3bd8212bb3d0458f80256514b8e31a03 2024-11-08T02:33:06,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c3e26b2dd4fe4364a7c269409f11cfb7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/c3e26b2dd4fe4364a7c269409f11cfb7 2024-11-08T02:33:06,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/06dbb542278643db8b5dce632414143d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/06dbb542278643db8b5dce632414143d 2024-11-08T02:33:06,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/084a617d4435474696a407d2879ef8e4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/084a617d4435474696a407d2879ef8e4 2024-11-08T02:33:06,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ea3963cb29f742848b19f6d397c7aba2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/ea3963cb29f742848b19f6d397c7aba2 2024-11-08T02:33:06,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/8d2579fe10a24c7c8226e67e6276c059 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/8d2579fe10a24c7c8226e67e6276c059 2024-11-08T02:33:06,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0cade48a41ec4d7abc76d0e73924533b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/0cade48a41ec4d7abc76d0e73924533b 2024-11-08T02:33:06,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6f32dc3cd4514e02b204e78bf07e9472, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/18f098f598824145a526dddc011f742a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b7fd0be761b944ea9e86f48888ed1c1e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6a2073863ae84927b67ec21756c2e793, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/90fd399fe82a44b3bad1cef4d0c1f66e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/686c9151c6134112bcdc2ea8d9fd536b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a986152e5e17450f9de838388fbf653b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/5effc6cbe93b49f891b1d4d71b16e66f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/20c441f326a04bb7a76af34180e9f0c6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ffb8b1f9f89c4c349b4c373f85a728b2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/13a976792d6c4eb8839c93a83e2c344d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b272771532414eaaa3b8663c555f7b04, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/86584502c94d4859af1d5db4d13e630f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ad77bba8be174468b77a6d1d53080e52, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8667f420f3a2490e96f2d8c752cbff19, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/bb7aab2ca92e43259dabf5181046f625, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a42ca26e073c47b09a2edae9b37a955a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8a22e5ebc1de41fd9da16d916630f84f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/991b533ad0e342f799d16f978ea2c21d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/847b7fb6e9344097afc7c5a629453b14, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4b36c0197c864ae8beb7a026a8da9810, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/f758b866aaec47418c5cdd6b4bd65a10, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/205da49022ac4652a7dc35471e768f03, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6ab64263c4844fc594af0111cd86e6f2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8dd71c1bcffc42529bb84ae761e5bd94] to archive 2024-11-08T02:33:06,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:33:06,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6f32dc3cd4514e02b204e78bf07e9472 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6f32dc3cd4514e02b204e78bf07e9472 2024-11-08T02:33:06,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/18f098f598824145a526dddc011f742a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/18f098f598824145a526dddc011f742a 2024-11-08T02:33:06,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b7fd0be761b944ea9e86f48888ed1c1e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b7fd0be761b944ea9e86f48888ed1c1e 2024-11-08T02:33:06,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6a2073863ae84927b67ec21756c2e793 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6a2073863ae84927b67ec21756c2e793 2024-11-08T02:33:06,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/90fd399fe82a44b3bad1cef4d0c1f66e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/90fd399fe82a44b3bad1cef4d0c1f66e 2024-11-08T02:33:06,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/686c9151c6134112bcdc2ea8d9fd536b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/686c9151c6134112bcdc2ea8d9fd536b 2024-11-08T02:33:06,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a986152e5e17450f9de838388fbf653b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a986152e5e17450f9de838388fbf653b 2024-11-08T02:33:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/5effc6cbe93b49f891b1d4d71b16e66f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/5effc6cbe93b49f891b1d4d71b16e66f 2024-11-08T02:33:06,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/20c441f326a04bb7a76af34180e9f0c6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/20c441f326a04bb7a76af34180e9f0c6 2024-11-08T02:33:06,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ffb8b1f9f89c4c349b4c373f85a728b2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ffb8b1f9f89c4c349b4c373f85a728b2 2024-11-08T02:33:06,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/13a976792d6c4eb8839c93a83e2c344d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/13a976792d6c4eb8839c93a83e2c344d 2024-11-08T02:33:06,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b272771532414eaaa3b8663c555f7b04 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/b272771532414eaaa3b8663c555f7b04 2024-11-08T02:33:06,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/86584502c94d4859af1d5db4d13e630f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/86584502c94d4859af1d5db4d13e630f 2024-11-08T02:33:06,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ad77bba8be174468b77a6d1d53080e52 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/ad77bba8be174468b77a6d1d53080e52 2024-11-08T02:33:06,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8667f420f3a2490e96f2d8c752cbff19 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8667f420f3a2490e96f2d8c752cbff19 2024-11-08T02:33:06,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/bb7aab2ca92e43259dabf5181046f625 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/bb7aab2ca92e43259dabf5181046f625 2024-11-08T02:33:06,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a42ca26e073c47b09a2edae9b37a955a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/a42ca26e073c47b09a2edae9b37a955a 2024-11-08T02:33:06,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8a22e5ebc1de41fd9da16d916630f84f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8a22e5ebc1de41fd9da16d916630f84f 2024-11-08T02:33:06,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/991b533ad0e342f799d16f978ea2c21d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/991b533ad0e342f799d16f978ea2c21d 2024-11-08T02:33:06,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/847b7fb6e9344097afc7c5a629453b14 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/847b7fb6e9344097afc7c5a629453b14 2024-11-08T02:33:06,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4b36c0197c864ae8beb7a026a8da9810 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4b36c0197c864ae8beb7a026a8da9810 2024-11-08T02:33:06,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/f758b866aaec47418c5cdd6b4bd65a10 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/f758b866aaec47418c5cdd6b4bd65a10 2024-11-08T02:33:06,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/205da49022ac4652a7dc35471e768f03 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/205da49022ac4652a7dc35471e768f03 2024-11-08T02:33:06,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6ab64263c4844fc594af0111cd86e6f2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/6ab64263c4844fc594af0111cd86e6f2 2024-11-08T02:33:06,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8dd71c1bcffc42529bb84ae761e5bd94 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/8dd71c1bcffc42529bb84ae761e5bd94 2024-11-08T02:33:06,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/543818838e2d4db8bd496c53409f5af1, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0fe325045cb64dfd8df8173b7b9ed6f3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3a31c1f0d58b4752956aa28d5874b663, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/4f833124d4374441a57bde8dc570f9d6, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/eed68c339a054b6f9da429ef770c4f27, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/81e01757b00b4994bb1caab50426596e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/308344520ff441b5b16b15b90fcdcc55, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/5f26b310ff01416b96e3e368c87079b4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/7c061783bd0847f48a94dc8c45afd696, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/07a9c6d46d94475ba8b7662abdfd9c02, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d1892fdf2a54469b965d0c61ef2a9ad0, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/12644c161bcf4839ae5e213b5b394c2b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ba552a832ba04c1584448990264ed2e4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/f874d6f6dcf74b1f97cbcaa2157d27c3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/248ca8677e6a44ca898ebe38615dc68c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0dc381c2138c49f2bbbed28a06eebce7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ca1d7dd9f8884853ac1b2c394fec8098, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/b9a65fa7c1fd4cf7b21e64921d941910, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3ba58a34152c4c098a86a7fb51ff251a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/6f40a3f4c4c840f5b7b56b724822728a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/97b81da9998b4091a17e814e5dbdbd38, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d0c8fe1d20f54b7ebcc71aa8d218ac0e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/792724f421bc4c8ea3da85e0166f53c7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ffe2cc8bcd8b4cce970fa5802099c4dc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/fc1c7765592144b9ad0a4dab767d113a] to archive 2024-11-08T02:33:06,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:33:06,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/543818838e2d4db8bd496c53409f5af1 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/543818838e2d4db8bd496c53409f5af1 2024-11-08T02:33:06,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0fe325045cb64dfd8df8173b7b9ed6f3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0fe325045cb64dfd8df8173b7b9ed6f3 2024-11-08T02:33:06,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3a31c1f0d58b4752956aa28d5874b663 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3a31c1f0d58b4752956aa28d5874b663 2024-11-08T02:33:06,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/4f833124d4374441a57bde8dc570f9d6 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/4f833124d4374441a57bde8dc570f9d6 2024-11-08T02:33:06,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/eed68c339a054b6f9da429ef770c4f27 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/eed68c339a054b6f9da429ef770c4f27 2024-11-08T02:33:06,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/81e01757b00b4994bb1caab50426596e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/81e01757b00b4994bb1caab50426596e 2024-11-08T02:33:06,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/308344520ff441b5b16b15b90fcdcc55 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/308344520ff441b5b16b15b90fcdcc55 2024-11-08T02:33:06,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/5f26b310ff01416b96e3e368c87079b4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/5f26b310ff01416b96e3e368c87079b4 2024-11-08T02:33:06,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/7c061783bd0847f48a94dc8c45afd696 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/7c061783bd0847f48a94dc8c45afd696 2024-11-08T02:33:06,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/07a9c6d46d94475ba8b7662abdfd9c02 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/07a9c6d46d94475ba8b7662abdfd9c02 2024-11-08T02:33:06,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d1892fdf2a54469b965d0c61ef2a9ad0 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d1892fdf2a54469b965d0c61ef2a9ad0 2024-11-08T02:33:06,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/12644c161bcf4839ae5e213b5b394c2b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/12644c161bcf4839ae5e213b5b394c2b 2024-11-08T02:33:06,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ba552a832ba04c1584448990264ed2e4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ba552a832ba04c1584448990264ed2e4 2024-11-08T02:33:06,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/f874d6f6dcf74b1f97cbcaa2157d27c3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/f874d6f6dcf74b1f97cbcaa2157d27c3 2024-11-08T02:33:06,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/248ca8677e6a44ca898ebe38615dc68c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/248ca8677e6a44ca898ebe38615dc68c 2024-11-08T02:33:06,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0dc381c2138c49f2bbbed28a06eebce7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/0dc381c2138c49f2bbbed28a06eebce7 2024-11-08T02:33:06,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ca1d7dd9f8884853ac1b2c394fec8098 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ca1d7dd9f8884853ac1b2c394fec8098 2024-11-08T02:33:06,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/b9a65fa7c1fd4cf7b21e64921d941910 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/b9a65fa7c1fd4cf7b21e64921d941910 2024-11-08T02:33:06,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3ba58a34152c4c098a86a7fb51ff251a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/3ba58a34152c4c098a86a7fb51ff251a 2024-11-08T02:33:06,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/6f40a3f4c4c840f5b7b56b724822728a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/6f40a3f4c4c840f5b7b56b724822728a 2024-11-08T02:33:06,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/97b81da9998b4091a17e814e5dbdbd38 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/97b81da9998b4091a17e814e5dbdbd38 2024-11-08T02:33:06,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d0c8fe1d20f54b7ebcc71aa8d218ac0e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/d0c8fe1d20f54b7ebcc71aa8d218ac0e 2024-11-08T02:33:06,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/792724f421bc4c8ea3da85e0166f53c7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/792724f421bc4c8ea3da85e0166f53c7 2024-11-08T02:33:06,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ffe2cc8bcd8b4cce970fa5802099c4dc to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/ffe2cc8bcd8b4cce970fa5802099c4dc 2024-11-08T02:33:06,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/fc1c7765592144b9ad0a4dab767d113a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/fc1c7765592144b9ad0a4dab767d113a 2024-11-08T02:33:06,833 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-08T02:33:06,833 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6. 2024-11-08T02:33:06,833 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] regionserver.HRegion(1635): Region close journal for b16fc799559c1038c4d1f04e11c99eb6: 2024-11-08T02:33:06,834 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=141}] handler.UnassignRegionHandler(170): Closed b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:33:06,835 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=b16fc799559c1038c4d1f04e11c99eb6, regionState=CLOSED 2024-11-08T02:33:06,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=141, resume processing ppid=140 2024-11-08T02:33:06,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, ppid=140, state=SUCCESS; CloseRegionProcedure b16fc799559c1038c4d1f04e11c99eb6, server=ecdbff65922a,40577,1731033026611 in 1.4940 sec 2024-11-08T02:33:06,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139 2024-11-08T02:33:06,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=b16fc799559c1038c4d1f04e11c99eb6, UNASSIGN in 1.4980 sec 2024-11-08T02:33:06,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=139, resume processing ppid=138 2024-11-08T02:33:06,838 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, ppid=138, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5010 sec 2024-11-08T02:33:06,839 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033186839"}]},"ts":"1731033186839"} 2024-11-08T02:33:06,840 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:33:06,879 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:33:06,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.5640 sec 2024-11-08T02:33:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-08T02:33:07,426 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 138 completed 2024-11-08T02:33:07,427 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:33:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=142, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:07,431 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=142, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:07,432 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=142, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-11-08T02:33:07,437 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:33:07,439 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/recovered.edits] 2024-11-08T02:33:07,441 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/21b7ef90495a491e98efc08b0dd0e193 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/21b7ef90495a491e98efc08b0dd0e193 2024-11-08T02:33:07,442 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/35e3bdf2bd9a46269c5c2ade02ed86b9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/35e3bdf2bd9a46269c5c2ade02ed86b9 2024-11-08T02:33:07,443 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4b2e153711804a94aa30bb4b65b71eeb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/A/4b2e153711804a94aa30bb4b65b71eeb 2024-11-08T02:33:07,445 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/099e735053db456b88f7747954785111 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/099e735053db456b88f7747954785111 2024-11-08T02:33:07,445 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4907739ce7df4517a1519e651655f16c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/4907739ce7df4517a1519e651655f16c 2024-11-08T02:33:07,446 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/7eaf5d815a994fbdaf3248b0a8ead188 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/B/7eaf5d815a994fbdaf3248b0a8ead188 2024-11-08T02:33:07,447 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/2a8455de7883468aa47f3c2200a353ec to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/2a8455de7883468aa47f3c2200a353ec 2024-11-08T02:33:07,448 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/448b47b0d4b2408eb435aa0bf3e07b5b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/448b47b0d4b2408eb435aa0bf3e07b5b 2024-11-08T02:33:07,449 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/949f52cede434c7d81991a5e4b2ff45a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/C/949f52cede434c7d81991a5e4b2ff45a 2024-11-08T02:33:07,450 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/recovered.edits/376.seqid to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6/recovered.edits/376.seqid 2024-11-08T02:33:07,451 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/b16fc799559c1038c4d1f04e11c99eb6 2024-11-08T02:33:07,451 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:33:07,452 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=142, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:07,453 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:33:07,454 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:33:07,455 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=142, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:07,455 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:33:07,455 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731033187455"}]},"ts":"9223372036854775807"} 2024-11-08T02:33:07,457 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-08T02:33:07,457 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => b16fc799559c1038c4d1f04e11c99eb6, NAME => 'TestAcidGuarantees,,1731033154577.b16fc799559c1038c4d1f04e11c99eb6.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:33:07,457 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:33:07,457 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731033187457"}]},"ts":"9223372036854775807"} 2024-11-08T02:33:07,458 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-08T02:33:07,505 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=142, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:07,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-08T02:33:07,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-11-08T02:33:07,536 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 142 completed 2024-11-08T02:33:07,551 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testScanAtomicity Thread=236 (was 238), OpenFileDescriptor=443 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=270 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=2095 (was 2117) 2024-11-08T02:33:07,561 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMobGetAtomicity Thread=236, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=270, ProcessCount=11, AvailableMemoryMB=2094 2024-11-08T02:33:07,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:33:07,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:33:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=143, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:07,564 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=143, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T02:33:07,564 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:07,564 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 143 2024-11-08T02:33:07,565 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=143, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T02:33:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-08T02:33:07,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742288_1464 (size=960) 2024-11-08T02:33:07,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-08T02:33:07,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-08T02:33:07,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643 2024-11-08T02:33:07,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742289_1465 (size=53) 2024-11-08T02:33:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-08T02:33:08,387 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:33:08,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 004f6c88876d7274b3d444aacc1fbc17, disabling compactions & flushes 2024-11-08T02:33:08,388 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. after waiting 0 ms 2024-11-08T02:33:08,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,388 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,388 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:08,390 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=143, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T02:33:08,391 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731033188390"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731033188390"}]},"ts":"1731033188390"} 2024-11-08T02:33:08,394 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-08T02:33:08,395 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=143, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T02:33:08,396 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033188395"}]},"ts":"1731033188395"} 2024-11-08T02:33:08,397 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T02:33:08,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, ASSIGN}] 2024-11-08T02:33:08,448 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, ASSIGN 2024-11-08T02:33:08,449 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, ASSIGN; state=OFFLINE, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=false 2024-11-08T02:33:08,600 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:08,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE; OpenRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:33:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-08T02:33:08,756 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:08,762 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,762 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7285): Opening region: {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:33:08,763 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,763 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:33:08,763 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7327): checking encryption for 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,763 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7330): checking classloading for 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,765 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,767 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:33:08,767 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 004f6c88876d7274b3d444aacc1fbc17 columnFamilyName A 2024-11-08T02:33:08,768 DEBUG [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:08,769 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(327): Store=004f6c88876d7274b3d444aacc1fbc17/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:33:08,769 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,770 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:33:08,770 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 004f6c88876d7274b3d444aacc1fbc17 columnFamilyName B 2024-11-08T02:33:08,771 DEBUG [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:08,771 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(327): Store=004f6c88876d7274b3d444aacc1fbc17/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:33:08,771 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,772 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:33:08,773 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 004f6c88876d7274b3d444aacc1fbc17 columnFamilyName C 2024-11-08T02:33:08,773 DEBUG [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:08,773 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(327): Store=004f6c88876d7274b3d444aacc1fbc17/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:33:08,773 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,774 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,775 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,777 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:33:08,778 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1085): writing seq id for 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:08,781 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T02:33:08,781 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1102): Opened 004f6c88876d7274b3d444aacc1fbc17; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73948650, jitterRate=0.10192075371742249}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:33:08,782 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1001): Region open journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:08,783 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., pid=145, masterSystemTime=1731033188756 2024-11-08T02:33:08,785 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,785 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:08,785 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=OPEN, openSeqNum=2, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:08,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=145, resume processing ppid=144 2024-11-08T02:33:08,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, ppid=144, state=SUCCESS; OpenRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 in 184 msec 2024-11-08T02:33:08,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-11-08T02:33:08,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, ASSIGN in 342 msec 2024-11-08T02:33:08,790 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=143, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T02:33:08,791 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033188791"}]},"ts":"1731033188791"} 2024-11-08T02:33:08,792 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T02:33:08,803 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=143, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T02:33:08,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2410 sec 2024-11-08T02:33:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-08T02:33:09,676 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 143 completed 2024-11-08T02:33:09,679 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2ac53e79 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d5efb7a 2024-11-08T02:33:09,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@644b7e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:09,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:09,742 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52296, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:09,744 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T02:33:09,746 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52346, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T02:33:09,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T02:33:09,749 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T02:33:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=146, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:09,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742290_1466 (size=996) 2024-11-08T02:33:10,168 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.960 2024-11-08T02:33:10,168 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.996 2024-11-08T02:33:10,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:33:10,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, REOPEN/MOVE}] 2024-11-08T02:33:10,177 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, REOPEN/MOVE 2024-11-08T02:33:10,177 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=148 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,179 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:33:10,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE; CloseRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:33:10,331 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,332 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(124): Close 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,332 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:33:10,332 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1681): Closing 004f6c88876d7274b3d444aacc1fbc17, disabling compactions & flushes 2024-11-08T02:33:10,332 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,333 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,333 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. after waiting 0 ms 2024-11-08T02:33:10,333 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,341 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T02:33:10,342 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,342 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegion(1635): Region close journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:10,342 WARN [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] regionserver.HRegionServer(3786): Not adding moved region record: 004f6c88876d7274b3d444aacc1fbc17 to self. 2024-11-08T02:33:10,343 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=149}] handler.UnassignRegionHandler(170): Closed 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=148 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=CLOSED 2024-11-08T02:33:10,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=148 2024-11-08T02:33:10,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=148, state=SUCCESS; CloseRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 in 166 msec 2024-11-08T02:33:10,347 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=148, ppid=147, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, REOPEN/MOVE; state=CLOSED, location=ecdbff65922a,40577,1731033026611; forceNewPlan=false, retain=true 2024-11-08T02:33:10,497 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=148 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=OPENING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,498 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=150, ppid=148, state=RUNNABLE; OpenRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:33:10,650 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,654 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,654 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7285): Opening region: {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} 2024-11-08T02:33:10,654 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,655 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T02:33:10,655 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7327): checking encryption for 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,655 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(7330): checking classloading for 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,657 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,658 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:33:10,659 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 004f6c88876d7274b3d444aacc1fbc17 columnFamilyName A 2024-11-08T02:33:10,661 DEBUG [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:10,662 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(327): Store=004f6c88876d7274b3d444aacc1fbc17/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:33:10,662 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,663 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:33:10,663 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 004f6c88876d7274b3d444aacc1fbc17 columnFamilyName B 2024-11-08T02:33:10,663 DEBUG [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:10,664 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(327): Store=004f6c88876d7274b3d444aacc1fbc17/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:33:10,664 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,665 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-08T02:33:10,665 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 004f6c88876d7274b3d444aacc1fbc17 columnFamilyName C 2024-11-08T02:33:10,665 DEBUG [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:10,666 INFO [StoreOpener-004f6c88876d7274b3d444aacc1fbc17-1 {}] regionserver.HStore(327): Store=004f6c88876d7274b3d444aacc1fbc17/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T02:33:10,666 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,666 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,667 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,669 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T02:33:10,670 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1085): writing seq id for 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,671 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1102): Opened 004f6c88876d7274b3d444aacc1fbc17; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59621744, jitterRate=-0.11156678199768066}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T02:33:10,671 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegion(1001): Region open journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:10,672 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., pid=150, masterSystemTime=1731033190650 2024-11-08T02:33:10,673 DEBUG [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,673 INFO [RS_OPEN_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_OPEN_REGION, pid=150}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=148 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=OPEN, openSeqNum=5, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=150, resume processing ppid=148 2024-11-08T02:33:10,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=150, ppid=148, state=SUCCESS; OpenRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 in 176 msec 2024-11-08T02:33:10,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=148, resume processing ppid=147 2024-11-08T02:33:10,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=148, ppid=147, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, REOPEN/MOVE in 499 msec 2024-11-08T02:33:10,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=147, resume processing ppid=146 2024-11-08T02:33:10,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, ppid=146, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 503 msec 2024-11-08T02:33:10,678 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 928 msec 2024-11-08T02:33:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=146 2024-11-08T02:33:10,680 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05bc9c3e to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7fc332d8 2024-11-08T02:33:10,721 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9b5141, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,722 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7181df3b to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@17327621 2024-11-08T02:33:10,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11a52cdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,730 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11030ef5 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1584f18a 2024-11-08T02:33:10,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7fe431, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,738 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69abefea to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5b914bf4 2024-11-08T02:33:10,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91d72db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,746 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e757135 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3f6a59e4 2024-11-08T02:33:10,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d836f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,754 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x150e08ed to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@53305d9b 2024-11-08T02:33:10,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11c440f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,762 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a3b66d3 to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bb6288a 2024-11-08T02:33:10,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58460ef3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,771 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5cfdf76c to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6556601 2024-11-08T02:33:10,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8cd1ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,780 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68c2838a to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@458a85fd 2024-11-08T02:33:10,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d832d43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,789 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50bf224f to 127.0.0.1:56877 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@410bf0c8 2024-11-08T02:33:10,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b6349f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T02:33:10,802 DEBUG [hconnection-0x2175a174-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,803 DEBUG [hconnection-0x7d3ef440-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,803 DEBUG [hconnection-0x46ebb0ff-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,803 DEBUG [hconnection-0x2fc17cc4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,804 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52308, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,804 DEBUG [hconnection-0x118a80cc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,804 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52318, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,804 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52334, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:10,805 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52340, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,805 DEBUG [hconnection-0x6dc7c449-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,805 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52346, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees 2024-11-08T02:33:10,806 DEBUG [hconnection-0xfe3430d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,806 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52350, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,806 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:10,807 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52362, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,807 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-08T02:33:10,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T02:33:10,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:10,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:10,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:10,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:10,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:10,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:10,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033250821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033250821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033250821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033250823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,825 DEBUG [hconnection-0x465cab1b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,826 DEBUG [hconnection-0x3d89c3c8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,826 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52372, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,826 DEBUG [hconnection-0x7022ecc1-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T02:33:10,827 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52374, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,827 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52390, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T02:33:10,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033250828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084bff7e216f35405aa830f7d6ab2f35d8_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_1/A:col10/1731033190811/Put/seqid=0 2024-11-08T02:33:10,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742291_1467 (size=9714) 2024-11-08T02:33:10,856 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:10,859 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084bff7e216f35405aa830f7d6ab2f35d8_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084bff7e216f35405aa830f7d6ab2f35d8_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:10,860 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/e0037c20383c4215a5ca34e67e181389, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:10,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/e0037c20383c4215a5ca34e67e181389 is 175, key is test_row_1/A:col10/1731033190811/Put/seqid=0 2024-11-08T02:33:10,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742292_1468 (size=22361) 2024-11-08T02:33:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-08T02:33:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033250924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033250924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033250924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033250924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033250929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,958 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:10,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-08T02:33:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:10,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:10,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-08T02:33:11,111 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-08T02:33:11,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:11,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033251126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033251126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033251127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033251127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033251132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,263 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-08T02:33:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/e0037c20383c4215a5ca34e67e181389 2024-11-08T02:33:11,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/8145b82ae4174af189bdd570f38c55d5 is 50, key is test_row_1/B:col10/1731033190811/Put/seqid=0 2024-11-08T02:33:11,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742293_1469 (size=9657) 2024-11-08T02:33:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-08T02:33:11,415 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-08T02:33:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:11,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033251428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033251429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033251429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033251430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033251434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,567 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-08T02:33:11,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:11,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/8145b82ae4174af189bdd570f38c55d5 2024-11-08T02:33:11,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/73ac849cb1e84893bef595a076c4f210 is 50, key is test_row_1/C:col10/1731033190811/Put/seqid=0 2024-11-08T02:33:11,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742294_1470 (size=9657) 2024-11-08T02:33:11,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/73ac849cb1e84893bef595a076c4f210 2024-11-08T02:33:11,719 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-08T02:33:11,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:11,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:11,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/e0037c20383c4215a5ca34e67e181389 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/e0037c20383c4215a5ca34e67e181389 2024-11-08T02:33:11,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/e0037c20383c4215a5ca34e67e181389, entries=100, sequenceid=17, filesize=21.8 K 2024-11-08T02:33:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/8145b82ae4174af189bdd570f38c55d5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/8145b82ae4174af189bdd570f38c55d5 2024-11-08T02:33:11,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/8145b82ae4174af189bdd570f38c55d5, entries=100, sequenceid=17, filesize=9.4 K 2024-11-08T02:33:11,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/73ac849cb1e84893bef595a076c4f210 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/73ac849cb1e84893bef595a076c4f210 2024-11-08T02:33:11,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/73ac849cb1e84893bef595a076c4f210, entries=100, sequenceid=17, filesize=9.4 K 2024-11-08T02:33:11,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 004f6c88876d7274b3d444aacc1fbc17 in 928ms, sequenceid=17, compaction requested=false 2024-11-08T02:33:11,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:11,871 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-08T02:33:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:11,872 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:33:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:11,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:11,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110863c02d13827f4604898801d450e41fee_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033190820/Put/seqid=0 2024-11-08T02:33:11,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742295_1471 (size=12154) 2024-11-08T02:33:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-08T02:33:11,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:11,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:11,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033251937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033251938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033251938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033251938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:11,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:11,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033251939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033252040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033252041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033252041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033252042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033252244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033252244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033252244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033252246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:12,284 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110863c02d13827f4604898801d450e41fee_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110863c02d13827f4604898801d450e41fee_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/49539e1815464d26b84461851a2d67bb, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/49539e1815464d26b84461851a2d67bb is 175, key is test_row_0/A:col10/1731033190820/Put/seqid=0 2024-11-08T02:33:12,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742296_1472 (size=30955) 2024-11-08T02:33:12,357 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T02:33:12,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033252546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033252547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033252547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033252548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:12,689 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/49539e1815464d26b84461851a2d67bb 2024-11-08T02:33:12,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/70959e2659b54717a0e28617798d4f96 is 50, key is test_row_0/B:col10/1731033190820/Put/seqid=0 2024-11-08T02:33:12,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742297_1473 (size=12001) 2024-11-08T02:33:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-08T02:33:12,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:12,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033252942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:13,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:13,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033253048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:13,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:13,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033253053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:13,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033253053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:13,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033253054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:13,099 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/70959e2659b54717a0e28617798d4f96 2024-11-08T02:33:13,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/371d0be258a54f1e84edac90b87373f2 is 50, key is test_row_0/C:col10/1731033190820/Put/seqid=0 2024-11-08T02:33:13,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742298_1474 (size=12001) 2024-11-08T02:33:13,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/371d0be258a54f1e84edac90b87373f2 2024-11-08T02:33:13,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/49539e1815464d26b84461851a2d67bb as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/49539e1815464d26b84461851a2d67bb 2024-11-08T02:33:13,123 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/49539e1815464d26b84461851a2d67bb, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T02:33:13,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/70959e2659b54717a0e28617798d4f96 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/70959e2659b54717a0e28617798d4f96 2024-11-08T02:33:13,127 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/70959e2659b54717a0e28617798d4f96, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:33:13,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/371d0be258a54f1e84edac90b87373f2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/371d0be258a54f1e84edac90b87373f2 2024-11-08T02:33:13,130 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/371d0be258a54f1e84edac90b87373f2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T02:33:13,130 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 004f6c88876d7274b3d444aacc1fbc17 in 1258ms, sequenceid=40, compaction requested=false 2024-11-08T02:33:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:13,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-11-08T02:33:13,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=152 2024-11-08T02:33:13,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=152, resume processing ppid=151 2024-11-08T02:33:13,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=152, ppid=151, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3240 sec 2024-11-08T02:33:13,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=151, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees in 2.3310 sec 2024-11-08T02:33:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:14,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:33:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:14,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:14,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108765bd0ad3adb4f16b840347c767cd68f_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033191934/Put/seqid=0 2024-11-08T02:33:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742299_1475 (size=12154) 2024-11-08T02:33:14,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033254074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033254075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033254077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033254077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033254177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033254178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033254179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033254180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033254380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033254381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033254382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033254383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:14,476 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108765bd0ad3adb4f16b840347c767cd68f_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108765bd0ad3adb4f16b840347c767cd68f_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:14,476 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3875ab90f4d74f4a87cc79a88a8b67d3, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:14,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3875ab90f4d74f4a87cc79a88a8b67d3 is 175, key is test_row_0/A:col10/1731033191934/Put/seqid=0 2024-11-08T02:33:14,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742300_1476 (size=30955) 2024-11-08T02:33:14,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033254683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033254685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033254685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033254687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3875ab90f4d74f4a87cc79a88a8b67d3 2024-11-08T02:33:14,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/1fac69e1693348bab64691a227e5a773 is 50, key is test_row_0/B:col10/1731033191934/Put/seqid=0 2024-11-08T02:33:14,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742301_1477 (size=12001) 2024-11-08T02:33:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-08T02:33:14,911 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 151 completed 2024-11-08T02:33:14,912 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=153, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees 2024-11-08T02:33:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-08T02:33:14,913 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=153, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:14,913 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=153, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:14,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:14,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033254961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:14,963 DEBUG [Thread-2106 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4163 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:33:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-08T02:33:15,065 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:15,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-08T02:33:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] handler.RSProcedureHandler(58): pid=154 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:15,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=154 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=154 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:15,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033255187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:15,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033255187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:15,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033255191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:15,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033255192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-08T02:33:15,217 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:15,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-08T02:33:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:15,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] handler.RSProcedureHandler(58): pid=154 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:15,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=154 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=154 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:15,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/1fac69e1693348bab64691a227e5a773 2024-11-08T02:33:15,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/5a6a855283ec452bb78dbca6dc600a31 is 50, key is test_row_0/C:col10/1731033191934/Put/seqid=0 2024-11-08T02:33:15,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742302_1478 (size=12001) 2024-11-08T02:33:15,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/5a6a855283ec452bb78dbca6dc600a31 2024-11-08T02:33:15,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3875ab90f4d74f4a87cc79a88a8b67d3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3875ab90f4d74f4a87cc79a88a8b67d3 2024-11-08T02:33:15,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3875ab90f4d74f4a87cc79a88a8b67d3, entries=150, sequenceid=54, filesize=30.2 K 2024-11-08T02:33:15,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/1fac69e1693348bab64691a227e5a773 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1fac69e1693348bab64691a227e5a773 2024-11-08T02:33:15,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1fac69e1693348bab64691a227e5a773, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:33:15,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/5a6a855283ec452bb78dbca6dc600a31 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/5a6a855283ec452bb78dbca6dc600a31 2024-11-08T02:33:15,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/5a6a855283ec452bb78dbca6dc600a31, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T02:33:15,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 004f6c88876d7274b3d444aacc1fbc17 in 1271ms, sequenceid=54, compaction requested=true 2024-11-08T02:33:15,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:33:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:15,330 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:33:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:15,330 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:33:15,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:15,330 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:15,330 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:15,330 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/A is initiating minor compaction (all files) 2024-11-08T02:33:15,330 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/B is initiating minor compaction (all files) 2024-11-08T02:33:15,330 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/A in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,330 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/B in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,331 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/e0037c20383c4215a5ca34e67e181389, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/49539e1815464d26b84461851a2d67bb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3875ab90f4d74f4a87cc79a88a8b67d3] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=82.3 K 2024-11-08T02:33:15,331 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/8145b82ae4174af189bdd570f38c55d5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/70959e2659b54717a0e28617798d4f96, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1fac69e1693348bab64691a227e5a773] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=32.9 K 2024-11-08T02:33:15,331 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,331 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/e0037c20383c4215a5ca34e67e181389, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/49539e1815464d26b84461851a2d67bb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3875ab90f4d74f4a87cc79a88a8b67d3] 2024-11-08T02:33:15,331 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 8145b82ae4174af189bdd570f38c55d5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731033190811 2024-11-08T02:33:15,331 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting e0037c20383c4215a5ca34e67e181389, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731033190811 2024-11-08T02:33:15,331 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 70959e2659b54717a0e28617798d4f96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033190820 2024-11-08T02:33:15,331 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 49539e1815464d26b84461851a2d67bb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033190820 2024-11-08T02:33:15,331 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1fac69e1693348bab64691a227e5a773, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033191934 2024-11-08T02:33:15,331 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3875ab90f4d74f4a87cc79a88a8b67d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033191934 2024-11-08T02:33:15,335 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:15,336 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:15,337 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/f9a6b3c4ea1b4bac90d5e77f527d3a99 is 50, key is test_row_0/B:col10/1731033191934/Put/seqid=0 2024-11-08T02:33:15,337 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108bf857a7069c645cdab8d46eba6514fd9_004f6c88876d7274b3d444aacc1fbc17 store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:15,338 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108bf857a7069c645cdab8d46eba6514fd9_004f6c88876d7274b3d444aacc1fbc17, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:15,339 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bf857a7069c645cdab8d46eba6514fd9_004f6c88876d7274b3d444aacc1fbc17 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:15,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742303_1479 (size=12104) 2024-11-08T02:33:15,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742304_1480 (size=4469) 2024-11-08T02:33:15,343 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/f9a6b3c4ea1b4bac90d5e77f527d3a99 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f9a6b3c4ea1b4bac90d5e77f527d3a99 2024-11-08T02:33:15,346 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/B of 004f6c88876d7274b3d444aacc1fbc17 into f9a6b3c4ea1b4bac90d5e77f527d3a99(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:15,346 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:15,346 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/B, priority=13, startTime=1731033195330; duration=0sec 2024-11-08T02:33:15,346 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:15,346 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:B 2024-11-08T02:33:15,346 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:15,347 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:15,347 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/C is initiating minor compaction (all files) 2024-11-08T02:33:15,347 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/C in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,347 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/73ac849cb1e84893bef595a076c4f210, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/371d0be258a54f1e84edac90b87373f2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/5a6a855283ec452bb78dbca6dc600a31] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=32.9 K 2024-11-08T02:33:15,347 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 73ac849cb1e84893bef595a076c4f210, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731033190811 2024-11-08T02:33:15,348 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 371d0be258a54f1e84edac90b87373f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731033190820 2024-11-08T02:33:15,348 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 5a6a855283ec452bb78dbca6dc600a31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033191934 2024-11-08T02:33:15,353 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:15,353 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/7121df4274724e3f8f7a4cc91ec3b6e2 is 50, key is test_row_0/C:col10/1731033191934/Put/seqid=0 2024-11-08T02:33:15,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742305_1481 (size=12104) 2024-11-08T02:33:15,369 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:15,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-08T02:33:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:15,369 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:33:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:15,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:15,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082eb613e8e99043d49bd179da1192c721_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033194076/Put/seqid=0 2024-11-08T02:33:15,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742306_1482 (size=12154) 2024-11-08T02:33:15,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:15,382 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082eb613e8e99043d49bd179da1192c721_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082eb613e8e99043d49bd179da1192c721_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:15,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4da8ca0d2fc34f5baeede8da52d884cf, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:15,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4da8ca0d2fc34f5baeede8da52d884cf is 175, key is test_row_0/A:col10/1731033194076/Put/seqid=0 2024-11-08T02:33:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742307_1483 (size=30955) 2024-11-08T02:33:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-08T02:33:15,742 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#A#compaction#406 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:15,743 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/93a4bba71d5343d6b5cf9bcafb79fdb4 is 175, key is test_row_0/A:col10/1731033191934/Put/seqid=0 2024-11-08T02:33:15,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742308_1484 (size=31058) 2024-11-08T02:33:15,760 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/7121df4274724e3f8f7a4cc91ec3b6e2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/7121df4274724e3f8f7a4cc91ec3b6e2 2024-11-08T02:33:15,763 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/C of 004f6c88876d7274b3d444aacc1fbc17 into 7121df4274724e3f8f7a4cc91ec3b6e2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:15,763 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:15,763 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/C, priority=13, startTime=1731033195330; duration=0sec 2024-11-08T02:33:15,763 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:15,763 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:C 2024-11-08T02:33:15,786 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4da8ca0d2fc34f5baeede8da52d884cf 2024-11-08T02:33:15,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b39d1ae771814726a9161ef847091e3d is 50, key is test_row_0/B:col10/1731033194076/Put/seqid=0 2024-11-08T02:33:15,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742309_1485 (size=12001) 2024-11-08T02:33:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-08T02:33:16,149 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/93a4bba71d5343d6b5cf9bcafb79fdb4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/93a4bba71d5343d6b5cf9bcafb79fdb4 2024-11-08T02:33:16,152 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/A of 004f6c88876d7274b3d444aacc1fbc17 into 93a4bba71d5343d6b5cf9bcafb79fdb4(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:16,152 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:16,152 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/A, priority=13, startTime=1731033195330; duration=0sec 2024-11-08T02:33:16,152 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:16,152 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:A 2024-11-08T02:33:16,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:16,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:16,205 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b39d1ae771814726a9161ef847091e3d 2024-11-08T02:33:16,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033256202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033256204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033256204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033256205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/1ab3b605982d4a13a8222fd566f568bd is 50, key is test_row_0/C:col10/1731033194076/Put/seqid=0 2024-11-08T02:33:16,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742310_1486 (size=12001) 2024-11-08T02:33:16,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033256306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033256307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033256308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033256308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033256508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033256510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033256511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033256511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,614 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/1ab3b605982d4a13a8222fd566f568bd 2024-11-08T02:33:16,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4da8ca0d2fc34f5baeede8da52d884cf as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4da8ca0d2fc34f5baeede8da52d884cf 2024-11-08T02:33:16,619 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4da8ca0d2fc34f5baeede8da52d884cf, entries=150, sequenceid=78, filesize=30.2 K 2024-11-08T02:33:16,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b39d1ae771814726a9161ef847091e3d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b39d1ae771814726a9161ef847091e3d 2024-11-08T02:33:16,622 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b39d1ae771814726a9161ef847091e3d, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T02:33:16,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/1ab3b605982d4a13a8222fd566f568bd as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1ab3b605982d4a13a8222fd566f568bd 2024-11-08T02:33:16,626 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1ab3b605982d4a13a8222fd566f568bd, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T02:33:16,626 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 004f6c88876d7274b3d444aacc1fbc17 in 1257ms, sequenceid=78, compaction requested=false 2024-11-08T02:33:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:16,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-11-08T02:33:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=154 2024-11-08T02:33:16,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=154, resume processing ppid=153 2024-11-08T02:33:16,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=154, ppid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7140 sec 2024-11-08T02:33:16,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees in 1.7160 sec 2024-11-08T02:33:16,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:16,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:33:16,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:16,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:16,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:16,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:16,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:16,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:16,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110813763a0388d54acea1f9f05308718935_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033196203/Put/seqid=0 2024-11-08T02:33:16,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742311_1487 (size=12154) 2024-11-08T02:33:16,821 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:16,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110813763a0388d54acea1f9f05308718935_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110813763a0388d54acea1f9f05308718935_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:16,825 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/713b93d475a44bd6a786ce562e0953a7, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:16,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/713b93d475a44bd6a786ce562e0953a7 is 175, key is test_row_0/A:col10/1731033196203/Put/seqid=0 2024-11-08T02:33:16,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742312_1488 (size=30955) 2024-11-08T02:33:16,829 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/713b93d475a44bd6a786ce562e0953a7 2024-11-08T02:33:16,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/d8feea37126643f4bb6a9506289fbbcc is 50, key is test_row_0/B:col10/1731033196203/Put/seqid=0 2024-11-08T02:33:16,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033256835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033256836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033256837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742313_1489 (size=12001) 2024-11-08T02:33:16,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033256837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033256938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033256940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033256940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033256941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-08T02:33:17,016 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 153 completed 2024-11-08T02:33:17,017 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:17,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=155, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees 2024-11-08T02:33:17,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-08T02:33:17,018 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=155, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:17,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=155, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:17,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:17,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-08T02:33:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033257140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033257143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033257143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033257144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,170 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-11-08T02:33:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:17,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] handler.RSProcedureHandler(58): pid=156 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:17,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=156 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:17,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=156 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:17,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/d8feea37126643f4bb6a9506289fbbcc 2024-11-08T02:33:17,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/76050f9ec91b4f2b81fa5916d215265f is 50, key is test_row_0/C:col10/1731033196203/Put/seqid=0 2024-11-08T02:33:17,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742314_1490 (size=12001) 2024-11-08T02:33:17,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/76050f9ec91b4f2b81fa5916d215265f 2024-11-08T02:33:17,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/713b93d475a44bd6a786ce562e0953a7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/713b93d475a44bd6a786ce562e0953a7 2024-11-08T02:33:17,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/713b93d475a44bd6a786ce562e0953a7, entries=150, sequenceid=94, filesize=30.2 K 2024-11-08T02:33:17,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/d8feea37126643f4bb6a9506289fbbcc as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d8feea37126643f4bb6a9506289fbbcc 2024-11-08T02:33:17,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d8feea37126643f4bb6a9506289fbbcc, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T02:33:17,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/76050f9ec91b4f2b81fa5916d215265f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/76050f9ec91b4f2b81fa5916d215265f 2024-11-08T02:33:17,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/76050f9ec91b4f2b81fa5916d215265f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T02:33:17,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 004f6c88876d7274b3d444aacc1fbc17 in 455ms, sequenceid=94, compaction requested=true 2024-11-08T02:33:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:33:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:17,268 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:33:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:17,268 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:33:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:17,268 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:17,268 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:17,269 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/B is initiating minor compaction (all files) 2024-11-08T02:33:17,269 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/A is initiating minor compaction (all files) 2024-11-08T02:33:17,269 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/B in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:17,269 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/A in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:17,269 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f9a6b3c4ea1b4bac90d5e77f527d3a99, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b39d1ae771814726a9161ef847091e3d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d8feea37126643f4bb6a9506289fbbcc] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=35.3 K 2024-11-08T02:33:17,269 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/93a4bba71d5343d6b5cf9bcafb79fdb4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4da8ca0d2fc34f5baeede8da52d884cf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/713b93d475a44bd6a786ce562e0953a7] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=90.8 K 2024-11-08T02:33:17,269 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:17,269 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/93a4bba71d5343d6b5cf9bcafb79fdb4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4da8ca0d2fc34f5baeede8da52d884cf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/713b93d475a44bd6a786ce562e0953a7] 2024-11-08T02:33:17,269 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f9a6b3c4ea1b4bac90d5e77f527d3a99, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033191934 2024-11-08T02:33:17,269 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 93a4bba71d5343d6b5cf9bcafb79fdb4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033191934 2024-11-08T02:33:17,269 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b39d1ae771814726a9161ef847091e3d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731033194074 2024-11-08T02:33:17,269 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 4da8ca0d2fc34f5baeede8da52d884cf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731033194074 2024-11-08T02:33:17,270 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d8feea37126643f4bb6a9506289fbbcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731033196203 2024-11-08T02:33:17,270 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 713b93d475a44bd6a786ce562e0953a7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731033196203 2024-11-08T02:33:17,274 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:17,275 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:17,276 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/a5c3a1a27ede4d21af6bf8620f4b8188 is 50, key is test_row_0/B:col10/1731033196203/Put/seqid=0 2024-11-08T02:33:17,277 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108b729340a326643fba74c74c145a27453_004f6c88876d7274b3d444aacc1fbc17 store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:17,279 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108b729340a326643fba74c74c145a27453_004f6c88876d7274b3d444aacc1fbc17, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:17,279 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b729340a326643fba74c74c145a27453_004f6c88876d7274b3d444aacc1fbc17 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:17,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742315_1491 (size=12207) 2024-11-08T02:33:17,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742316_1492 (size=4469) 2024-11-08T02:33:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-08T02:33:17,322 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-11-08T02:33:17,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:17,322 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T02:33:17,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:17,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:17,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:17,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f6413811cc434425a8a8e7a04666f2d6_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033196836/Put/seqid=0 2024-11-08T02:33:17,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742317_1493 (size=12154) 2024-11-08T02:33:17,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:17,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033257452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033257456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033257456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033257456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033257557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033257559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033257559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033257559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-08T02:33:17,684 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#A#compaction#415 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:17,685 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d12348139d6a4664b64a5d349db0a17c is 175, key is test_row_0/A:col10/1731033196203/Put/seqid=0 2024-11-08T02:33:17,686 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/a5c3a1a27ede4d21af6bf8620f4b8188 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a5c3a1a27ede4d21af6bf8620f4b8188 2024-11-08T02:33:17,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742318_1494 (size=31161) 2024-11-08T02:33:17,690 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/B of 004f6c88876d7274b3d444aacc1fbc17 into a5c3a1a27ede4d21af6bf8620f4b8188(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:17,690 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:17,690 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/B, priority=13, startTime=1731033197268; duration=0sec 2024-11-08T02:33:17,690 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:17,690 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:B 2024-11-08T02:33:17,691 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:17,691 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:17,691 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/C is initiating minor compaction (all files) 2024-11-08T02:33:17,691 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/C in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:17,691 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/7121df4274724e3f8f7a4cc91ec3b6e2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1ab3b605982d4a13a8222fd566f568bd, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/76050f9ec91b4f2b81fa5916d215265f] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=35.3 K 2024-11-08T02:33:17,692 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7121df4274724e3f8f7a4cc91ec3b6e2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731033191934 2024-11-08T02:33:17,692 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1ab3b605982d4a13a8222fd566f568bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731033194074 2024-11-08T02:33:17,692 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 76050f9ec91b4f2b81fa5916d215265f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731033196203 2024-11-08T02:33:17,693 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d12348139d6a4664b64a5d349db0a17c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d12348139d6a4664b64a5d349db0a17c 2024-11-08T02:33:17,696 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/A of 004f6c88876d7274b3d444aacc1fbc17 into d12348139d6a4664b64a5d349db0a17c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:17,696 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:17,696 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/A, priority=13, startTime=1731033197268; duration=0sec 2024-11-08T02:33:17,696 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:17,697 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:A 2024-11-08T02:33:17,699 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#C#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:17,699 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/32956c117ca648c690b78bb2fa936b86 is 50, key is test_row_0/C:col10/1731033196203/Put/seqid=0 2024-11-08T02:33:17,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742319_1495 (size=12207) 2024-11-08T02:33:17,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:17,742 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f6413811cc434425a8a8e7a04666f2d6_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f6413811cc434425a8a8e7a04666f2d6_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d78d972dfc9447c7879913d1ff262a32, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:17,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d78d972dfc9447c7879913d1ff262a32 is 175, key is test_row_0/A:col10/1731033196836/Put/seqid=0 2024-11-08T02:33:17,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742320_1496 (size=30955) 2024-11-08T02:33:17,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033257761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033257762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033257762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:17,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033257764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033258065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033258066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033258066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033258067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,106 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/32956c117ca648c690b78bb2fa936b86 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/32956c117ca648c690b78bb2fa936b86 2024-11-08T02:33:18,109 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/C of 004f6c88876d7274b3d444aacc1fbc17 into 32956c117ca648c690b78bb2fa936b86(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:18,109 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:18,109 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/C, priority=13, startTime=1731033197268; duration=0sec 2024-11-08T02:33:18,109 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:18,109 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:C 2024-11-08T02:33:18,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-08T02:33:18,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d78d972dfc9447c7879913d1ff262a32 2024-11-08T02:33:18,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/617649466c04435b8012de57bcf88e80 is 50, key is test_row_0/B:col10/1731033196836/Put/seqid=0 2024-11-08T02:33:18,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742321_1497 (size=12001) 2024-11-08T02:33:18,556 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/617649466c04435b8012de57bcf88e80 2024-11-08T02:33:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/8499864ea2fa466e9497b90e928c6e2a is 50, key is test_row_0/C:col10/1731033196836/Put/seqid=0 2024-11-08T02:33:18,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742322_1498 (size=12001) 2024-11-08T02:33:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033258568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033258570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033258570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033258571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,968 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/8499864ea2fa466e9497b90e928c6e2a 2024-11-08T02:33:18,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033258967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:18,969 DEBUG [Thread-2106 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8170 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:33:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d78d972dfc9447c7879913d1ff262a32 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d78d972dfc9447c7879913d1ff262a32 2024-11-08T02:33:18,974 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d78d972dfc9447c7879913d1ff262a32, entries=150, sequenceid=116, filesize=30.2 K 2024-11-08T02:33:18,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/617649466c04435b8012de57bcf88e80 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/617649466c04435b8012de57bcf88e80 2024-11-08T02:33:18,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/617649466c04435b8012de57bcf88e80, entries=150, sequenceid=116, filesize=11.7 K 2024-11-08T02:33:18,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/8499864ea2fa466e9497b90e928c6e2a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8499864ea2fa466e9497b90e928c6e2a 2024-11-08T02:33:18,980 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8499864ea2fa466e9497b90e928c6e2a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-08T02:33:18,981 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 004f6c88876d7274b3d444aacc1fbc17 in 1659ms, sequenceid=116, compaction requested=false 2024-11-08T02:33:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:18,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=156 2024-11-08T02:33:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=156 2024-11-08T02:33:18,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=156, resume processing ppid=155 2024-11-08T02:33:18,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=156, ppid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-11-08T02:33:18,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees in 1.9660 sec 2024-11-08T02:33:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-08T02:33:19,121 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 155 completed 2024-11-08T02:33:19,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=157, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees 2024-11-08T02:33:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-08T02:33:19,123 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=157, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:19,123 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=157, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:19,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-08T02:33:19,276 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-08T02:33:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:19,277 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:33:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:19,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081e6d3b31f07a47dc88a1e7cb42b754c2_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033197448/Put/seqid=0 2024-11-08T02:33:19,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742323_1499 (size=12204) 2024-11-08T02:33:19,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:19,289 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081e6d3b31f07a47dc88a1e7cb42b754c2_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081e6d3b31f07a47dc88a1e7cb42b754c2_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:19,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/b51da5dab1394c92a61f5aa8633adb10, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:19,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/b51da5dab1394c92a61f5aa8633adb10 is 175, key is test_row_0/A:col10/1731033197448/Put/seqid=0 2024-11-08T02:33:19,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742324_1500 (size=31005) 2024-11-08T02:33:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-08T02:33:19,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:19,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:19,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033259590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033259591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033259591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033259592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033259693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,695 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/b51da5dab1394c92a61f5aa8633adb10 2024-11-08T02:33:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033259695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033259695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033259695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/7ba0ad33817641398bb299efb068cb25 is 50, key is test_row_0/B:col10/1731033197448/Put/seqid=0 2024-11-08T02:33:19,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742325_1501 (size=12051) 2024-11-08T02:33:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-08T02:33:19,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033259896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033259898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033259899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:19,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033259899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,107 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/7ba0ad33817641398bb299efb068cb25 2024-11-08T02:33:20,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/478f6ae73ba74b64b37948253ae59617 is 50, key is test_row_0/C:col10/1731033197448/Put/seqid=0 2024-11-08T02:33:20,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742326_1502 (size=12051) 2024-11-08T02:33:20,134 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/478f6ae73ba74b64b37948253ae59617 2024-11-08T02:33:20,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/b51da5dab1394c92a61f5aa8633adb10 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/b51da5dab1394c92a61f5aa8633adb10 2024-11-08T02:33:20,139 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/b51da5dab1394c92a61f5aa8633adb10, entries=150, sequenceid=133, filesize=30.3 K 2024-11-08T02:33:20,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/7ba0ad33817641398bb299efb068cb25 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/7ba0ad33817641398bb299efb068cb25 2024-11-08T02:33:20,143 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/7ba0ad33817641398bb299efb068cb25, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T02:33:20,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/478f6ae73ba74b64b37948253ae59617 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/478f6ae73ba74b64b37948253ae59617 2024-11-08T02:33:20,146 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/478f6ae73ba74b64b37948253ae59617, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T02:33:20,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 004f6c88876d7274b3d444aacc1fbc17 in 870ms, sequenceid=133, compaction requested=true 2024-11-08T02:33:20,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:20,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=158 2024-11-08T02:33:20,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=158 2024-11-08T02:33:20,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=158, resume processing ppid=157 2024-11-08T02:33:20,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=158, ppid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0250 sec 2024-11-08T02:33:20,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees in 1.0270 sec 2024-11-08T02:33:20,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:20,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:33:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:20,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:20,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:20,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:20,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:20,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:20,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080c38274c63134e3286342baa0555571d_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033200200/Put/seqid=0 2024-11-08T02:33:20,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742327_1503 (size=12304) 2024-11-08T02:33:20,211 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:20,214 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080c38274c63134e3286342baa0555571d_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c38274c63134e3286342baa0555571d_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:20,215 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/2f8712da666c41f9866ed8fa96a8865f, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:20,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/2f8712da666c41f9866ed8fa96a8865f is 175, key is test_row_0/A:col10/1731033200200/Put/seqid=0 2024-11-08T02:33:20,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742328_1504 (size=31105) 2024-11-08T02:33:20,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-08T02:33:20,226 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 157 completed 2024-11-08T02:33:20,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=159, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees 2024-11-08T02:33:20,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-08T02:33:20,228 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=159, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:20,228 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=159, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:20,228 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:20,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033260210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033260254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033260254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033260255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-08T02:33:20,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033260355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033260359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033260360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033260360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,379 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:20,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:20,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-08T02:33:20,531 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:20,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033260558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033260562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033260562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033260563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,623 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/2f8712da666c41f9866ed8fa96a8865f 2024-11-08T02:33:20,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/ccef4cbaa58541d28961fc39cdad85f7 is 50, key is test_row_0/B:col10/1731033200200/Put/seqid=0 2024-11-08T02:33:20,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742329_1505 (size=12151) 2024-11-08T02:33:20,683 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-08T02:33:20,836 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033260862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033260864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033260865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:20,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033260867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,988 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:20,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:20,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/ccef4cbaa58541d28961fc39cdad85f7 2024-11-08T02:33:21,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e is 50, key is test_row_0/C:col10/1731033200200/Put/seqid=0 2024-11-08T02:33:21,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742330_1506 (size=12151) 2024-11-08T02:33:21,140 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,292 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-08T02:33:21,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033261364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033261369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033261370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:21,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033261370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,445 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:21,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] handler.RSProcedureHandler(58): pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=160 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=160 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:21,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e 2024-11-08T02:33:21,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/2f8712da666c41f9866ed8fa96a8865f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2f8712da666c41f9866ed8fa96a8865f 2024-11-08T02:33:21,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2f8712da666c41f9866ed8fa96a8865f, entries=150, sequenceid=156, filesize=30.4 K 2024-11-08T02:33:21,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/ccef4cbaa58541d28961fc39cdad85f7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/ccef4cbaa58541d28961fc39cdad85f7 2024-11-08T02:33:21,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/ccef4cbaa58541d28961fc39cdad85f7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T02:33:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e 2024-11-08T02:33:21,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-08T02:33:21,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 004f6c88876d7274b3d444aacc1fbc17 in 1278ms, sequenceid=156, compaction requested=true 2024-11-08T02:33:21,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:33:21,480 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:33:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:21,480 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:33:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:33:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:33:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/A is initiating minor compaction (all files) 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/B is initiating minor compaction (all files) 2024-11-08T02:33:21,481 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/A in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,481 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/B in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,481 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d12348139d6a4664b64a5d349db0a17c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d78d972dfc9447c7879913d1ff262a32, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/b51da5dab1394c92a61f5aa8633adb10, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2f8712da666c41f9866ed8fa96a8865f] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=121.3 K 2024-11-08T02:33:21,481 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a5c3a1a27ede4d21af6bf8620f4b8188, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/617649466c04435b8012de57bcf88e80, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/7ba0ad33817641398bb299efb068cb25, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/ccef4cbaa58541d28961fc39cdad85f7] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=47.3 K 2024-11-08T02:33:21,481 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d12348139d6a4664b64a5d349db0a17c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d78d972dfc9447c7879913d1ff262a32, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/b51da5dab1394c92a61f5aa8633adb10, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2f8712da666c41f9866ed8fa96a8865f] 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a5c3a1a27ede4d21af6bf8620f4b8188, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731033196203 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d12348139d6a4664b64a5d349db0a17c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731033196203 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 617649466c04435b8012de57bcf88e80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731033196835 2024-11-08T02:33:21,481 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d78d972dfc9447c7879913d1ff262a32, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731033196835 2024-11-08T02:33:21,482 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 7ba0ad33817641398bb299efb068cb25, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731033197448 2024-11-08T02:33:21,482 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting b51da5dab1394c92a61f5aa8633adb10, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731033197448 2024-11-08T02:33:21,482 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting ccef4cbaa58541d28961fc39cdad85f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033199584 2024-11-08T02:33:21,482 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2f8712da666c41f9866ed8fa96a8865f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033199584 2024-11-08T02:33:21,487 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:21,488 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#B#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:21,488 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/cd7afcef66ee4255aa086b008bade5c4 is 50, key is test_row_0/B:col10/1731033200200/Put/seqid=0 2024-11-08T02:33:21,488 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411084ef17a4c64af4f6693f7567cd2c5c93e_004f6c88876d7274b3d444aacc1fbc17 store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:21,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742331_1507 (size=12493) 2024-11-08T02:33:21,491 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411084ef17a4c64af4f6693f7567cd2c5c93e_004f6c88876d7274b3d444aacc1fbc17, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:21,491 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084ef17a4c64af4f6693f7567cd2c5c93e_004f6c88876d7274b3d444aacc1fbc17 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:21,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742332_1508 (size=4469) 2024-11-08T02:33:21,597 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:21,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=160 2024-11-08T02:33:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,598 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T02:33:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:21,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:21,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110898b5a3e4ab164a22a1cef430030b062c_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033200215/Put/seqid=0 2024-11-08T02:33:21,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742333_1509 (size=12304) 2024-11-08T02:33:21,894 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/cd7afcef66ee4255aa086b008bade5c4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/cd7afcef66ee4255aa086b008bade5c4 2024-11-08T02:33:21,895 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#A#compaction#427 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:21,896 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/fa69ddcc15eb4da8abe5efb45f1a3437 is 175, key is test_row_0/A:col10/1731033200200/Put/seqid=0 2024-11-08T02:33:21,898 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/B of 004f6c88876d7274b3d444aacc1fbc17 into cd7afcef66ee4255aa086b008bade5c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:21,898 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:21,898 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/B, priority=12, startTime=1731033201480; duration=0sec 2024-11-08T02:33:21,898 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:21,898 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:B 2024-11-08T02:33:21,898 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T02:33:21,899 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T02:33:21,899 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/C is initiating minor compaction (all files) 2024-11-08T02:33:21,899 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/C in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:21,899 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/32956c117ca648c690b78bb2fa936b86, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8499864ea2fa466e9497b90e928c6e2a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/478f6ae73ba74b64b37948253ae59617, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=47.3 K 2024-11-08T02:33:21,899 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 32956c117ca648c690b78bb2fa936b86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731033196203 2024-11-08T02:33:21,900 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 8499864ea2fa466e9497b90e928c6e2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731033196835 2024-11-08T02:33:21,900 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 478f6ae73ba74b64b37948253ae59617, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731033197448 2024-11-08T02:33:21,900 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b40e6621d2fd4ff3ae9bd5202e4b1f1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033199584 2024-11-08T02:33:21,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742334_1510 (size=31447) 2024-11-08T02:33:21,908 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:21,908 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/cfb1be0b55574a4793db8e4ba682a4df is 50, key is test_row_0/C:col10/1731033200200/Put/seqid=0 2024-11-08T02:33:21,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742335_1511 (size=12493) 2024-11-08T02:33:21,915 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/cfb1be0b55574a4793db8e4ba682a4df as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/cfb1be0b55574a4793db8e4ba682a4df 2024-11-08T02:33:21,919 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/fa69ddcc15eb4da8abe5efb45f1a3437 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/fa69ddcc15eb4da8abe5efb45f1a3437 2024-11-08T02:33:21,919 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/C of 004f6c88876d7274b3d444aacc1fbc17 into cfb1be0b55574a4793db8e4ba682a4df(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:21,919 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:21,919 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/C, priority=12, startTime=1731033201480; duration=0sec 2024-11-08T02:33:21,919 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:21,919 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:C 2024-11-08T02:33:21,922 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/A of 004f6c88876d7274b3d444aacc1fbc17 into fa69ddcc15eb4da8abe5efb45f1a3437(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:21,922 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:21,922 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/A, priority=12, startTime=1731033201479; duration=0sec 2024-11-08T02:33:21,922 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:21,922 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:A 2024-11-08T02:33:22,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:22,017 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110898b5a3e4ab164a22a1cef430030b062c_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110898b5a3e4ab164a22a1cef430030b062c_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:22,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3928c5f090b74538bb0e1d472cf146a8, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:22,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3928c5f090b74538bb0e1d472cf146a8 is 175, key is test_row_0/A:col10/1731033200215/Put/seqid=0 2024-11-08T02:33:22,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742336_1512 (size=31105) 2024-11-08T02:33:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-08T02:33:22,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:22,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033262385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033262386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033262387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033262387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,423 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3928c5f090b74538bb0e1d472cf146a8 2024-11-08T02:33:22,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/493802baa7cf446cbcc32d02c0b46a0d is 50, key is test_row_0/B:col10/1731033200215/Put/seqid=0 2024-11-08T02:33:22,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742337_1513 (size=12151) 2024-11-08T02:33:22,436 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/493802baa7cf446cbcc32d02c0b46a0d 2024-11-08T02:33:22,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/0f30ab661f5548bcb2d8c013098bdca7 is 50, key is test_row_0/C:col10/1731033200215/Put/seqid=0 2024-11-08T02:33:22,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742338_1514 (size=12151) 2024-11-08T02:33:22,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033262488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033262489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033262490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033262490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033262691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033262692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033262692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:22,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033262692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:22,845 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/0f30ab661f5548bcb2d8c013098bdca7 2024-11-08T02:33:22,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3928c5f090b74538bb0e1d472cf146a8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3928c5f090b74538bb0e1d472cf146a8 2024-11-08T02:33:22,850 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3928c5f090b74538bb0e1d472cf146a8, entries=150, sequenceid=170, filesize=30.4 K 2024-11-08T02:33:22,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/493802baa7cf446cbcc32d02c0b46a0d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/493802baa7cf446cbcc32d02c0b46a0d 2024-11-08T02:33:22,854 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/493802baa7cf446cbcc32d02c0b46a0d, entries=150, sequenceid=170, filesize=11.9 K 2024-11-08T02:33:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/0f30ab661f5548bcb2d8c013098bdca7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/0f30ab661f5548bcb2d8c013098bdca7 2024-11-08T02:33:22,857 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/0f30ab661f5548bcb2d8c013098bdca7, entries=150, sequenceid=170, filesize=11.9 K 2024-11-08T02:33:22,857 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 004f6c88876d7274b3d444aacc1fbc17 in 1259ms, sequenceid=170, compaction requested=false 2024-11-08T02:33:22,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:22,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:22,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=160 2024-11-08T02:33:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=160 2024-11-08T02:33:22,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=160, resume processing ppid=159 2024-11-08T02:33:22,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=160, ppid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6300 sec 2024-11-08T02:33:22,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=159, table=TestAcidGuarantees in 2.6320 sec 2024-11-08T02:33:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:22,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T02:33:22,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:22,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:22,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:22,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:22,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:22,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:23,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e6f09ed820664fe5a238eec6b6822b40_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033202995/Put/seqid=0 2024-11-08T02:33:23,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742339_1515 (size=12304) 2024-11-08T02:33:23,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033263002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033263003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033263004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033263005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033263106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033263106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033263107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033263108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033263308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033263309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033263309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033263310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,406 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:23,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e6f09ed820664fe5a238eec6b6822b40_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e6f09ed820664fe5a238eec6b6822b40_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:23,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/f610108eec6d46e5822fba1c2f382905, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:23,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/f610108eec6d46e5822fba1c2f382905 is 175, key is test_row_0/A:col10/1731033202995/Put/seqid=0 2024-11-08T02:33:23,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742340_1516 (size=31105) 2024-11-08T02:33:23,413 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/f610108eec6d46e5822fba1c2f382905 2024-11-08T02:33:23,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b9b8cd4834d845acb74d581d4bd290d9 is 50, key is test_row_0/B:col10/1731033202995/Put/seqid=0 2024-11-08T02:33:23,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742341_1517 (size=12151) 2024-11-08T02:33:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033263611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033263612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033263613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033263613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:23,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b9b8cd4834d845acb74d581d4bd290d9 2024-11-08T02:33:23,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/f188ecdae7ac44d0aea651cb89a4fe05 is 50, key is test_row_0/C:col10/1731033202995/Put/seqid=0 2024-11-08T02:33:23,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742342_1518 (size=12151) 2024-11-08T02:33:24,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033264115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:24,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033264116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:24,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033264116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:24,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033264120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:24,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/f188ecdae7ac44d0aea651cb89a4fe05 2024-11-08T02:33:24,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/f610108eec6d46e5822fba1c2f382905 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/f610108eec6d46e5822fba1c2f382905 2024-11-08T02:33:24,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/f610108eec6d46e5822fba1c2f382905, entries=150, sequenceid=196, filesize=30.4 K 2024-11-08T02:33:24,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b9b8cd4834d845acb74d581d4bd290d9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b9b8cd4834d845acb74d581d4bd290d9 2024-11-08T02:33:24,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b9b8cd4834d845acb74d581d4bd290d9, entries=150, sequenceid=196, filesize=11.9 K 2024-11-08T02:33:24,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/f188ecdae7ac44d0aea651cb89a4fe05 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/f188ecdae7ac44d0aea651cb89a4fe05 2024-11-08T02:33:24,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/f188ecdae7ac44d0aea651cb89a4fe05, entries=150, sequenceid=196, filesize=11.9 K 2024-11-08T02:33:24,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 004f6c88876d7274b3d444aacc1fbc17 in 1247ms, sequenceid=196, compaction requested=true 2024-11-08T02:33:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:33:24,244 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:33:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:33:24,244 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:24,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:24,244 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:24,244 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/B is initiating minor compaction (all files) 2024-11-08T02:33:24,245 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/B in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:24,245 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/cd7afcef66ee4255aa086b008bade5c4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/493802baa7cf446cbcc32d02c0b46a0d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b9b8cd4834d845acb74d581d4bd290d9] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=35.9 K 2024-11-08T02:33:24,245 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:24,245 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/A is initiating minor compaction (all files) 2024-11-08T02:33:24,245 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/A in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:24,245 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/fa69ddcc15eb4da8abe5efb45f1a3437, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3928c5f090b74538bb0e1d472cf146a8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/f610108eec6d46e5822fba1c2f382905] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=91.5 K 2024-11-08T02:33:24,245 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:24,245 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/fa69ddcc15eb4da8abe5efb45f1a3437, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3928c5f090b74538bb0e1d472cf146a8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/f610108eec6d46e5822fba1c2f382905] 2024-11-08T02:33:24,245 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting cd7afcef66ee4255aa086b008bade5c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033199584 2024-11-08T02:33:24,245 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting fa69ddcc15eb4da8abe5efb45f1a3437, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033199584 2024-11-08T02:33:24,245 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 493802baa7cf446cbcc32d02c0b46a0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731033200209 2024-11-08T02:33:24,246 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3928c5f090b74538bb0e1d472cf146a8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731033200209 2024-11-08T02:33:24,246 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b9b8cd4834d845acb74d581d4bd290d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731033202381 2024-11-08T02:33:24,246 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting f610108eec6d46e5822fba1c2f382905, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731033202381 2024-11-08T02:33:24,250 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:24,252 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108e4198553ef364511a9dfc0a400dcad38_004f6c88876d7274b3d444aacc1fbc17 store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:24,252 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:24,252 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/969b1d93ff294d60a5b839bf5ad3053c is 50, key is test_row_0/B:col10/1731033202995/Put/seqid=0 2024-11-08T02:33:24,253 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108e4198553ef364511a9dfc0a400dcad38_004f6c88876d7274b3d444aacc1fbc17, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:24,253 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e4198553ef364511a9dfc0a400dcad38_004f6c88876d7274b3d444aacc1fbc17 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:24,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742343_1519 (size=12595) 2024-11-08T02:33:24,260 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/969b1d93ff294d60a5b839bf5ad3053c as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/969b1d93ff294d60a5b839bf5ad3053c 2024-11-08T02:33:24,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742344_1520 (size=4469) 2024-11-08T02:33:24,264 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/B of 004f6c88876d7274b3d444aacc1fbc17 into 969b1d93ff294d60a5b839bf5ad3053c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:24,264 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:24,264 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/B, priority=13, startTime=1731033204244; duration=0sec 2024-11-08T02:33:24,265 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:24,265 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:B 2024-11-08T02:33:24,265 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:24,265 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:24,265 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/C is initiating minor compaction (all files) 2024-11-08T02:33:24,265 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/C in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:24,265 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/cfb1be0b55574a4793db8e4ba682a4df, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/0f30ab661f5548bcb2d8c013098bdca7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/f188ecdae7ac44d0aea651cb89a4fe05] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=35.9 K 2024-11-08T02:33:24,266 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting cfb1be0b55574a4793db8e4ba682a4df, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731033199584 2024-11-08T02:33:24,266 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 0f30ab661f5548bcb2d8c013098bdca7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731033200209 2024-11-08T02:33:24,266 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f188ecdae7ac44d0aea651cb89a4fe05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731033202381 2024-11-08T02:33:24,271 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#C#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:24,272 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/b204e627bf9642c8baa3adbbb27df672 is 50, key is test_row_0/C:col10/1731033202995/Put/seqid=0 2024-11-08T02:33:24,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742345_1521 (size=12595) 2024-11-08T02:33:24,281 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/b204e627bf9642c8baa3adbbb27df672 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b204e627bf9642c8baa3adbbb27df672 2024-11-08T02:33:24,285 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/C of 004f6c88876d7274b3d444aacc1fbc17 into b204e627bf9642c8baa3adbbb27df672(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:24,285 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:24,285 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/C, priority=13, startTime=1731033204244; duration=0sec 2024-11-08T02:33:24,285 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:24,285 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:C 2024-11-08T02:33:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-08T02:33:24,331 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 159 completed 2024-11-08T02:33:24,332 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=161, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees 2024-11-08T02:33:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-08T02:33:24,333 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=161, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:24,333 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=161, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:24,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-08T02:33:24,484 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:24,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=162 2024-11-08T02:33:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:24,485 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T02:33:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e15c603861894ceb9f05800357a644f2_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033203002/Put/seqid=0 2024-11-08T02:33:24,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742346_1522 (size=12304) 2024-11-08T02:33:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-08T02:33:24,666 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#A#compaction#436 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:24,666 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/7bd8d230b4794e50a2d360b4f7fb01ed is 175, key is test_row_0/A:col10/1731033202995/Put/seqid=0 2024-11-08T02:33:24,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742347_1523 (size=31549) 2024-11-08T02:33:24,672 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/7bd8d230b4794e50a2d360b4f7fb01ed as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/7bd8d230b4794e50a2d360b4f7fb01ed 2024-11-08T02:33:24,676 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/A of 004f6c88876d7274b3d444aacc1fbc17 into 7bd8d230b4794e50a2d360b4f7fb01ed(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:24,676 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:24,676 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/A, priority=13, startTime=1731033204244; duration=0sec 2024-11-08T02:33:24,676 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:24,676 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:A 2024-11-08T02:33:24,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:24,896 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e15c603861894ceb9f05800357a644f2_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e15c603861894ceb9f05800357a644f2_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:24,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3ee432405bda4f9ab005caae0578417e, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:24,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3ee432405bda4f9ab005caae0578417e is 175, key is test_row_0/A:col10/1731033203002/Put/seqid=0 2024-11-08T02:33:24,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742348_1524 (size=31105) 2024-11-08T02:33:24,921 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3ee432405bda4f9ab005caae0578417e 2024-11-08T02:33:24,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/c8d537f21cf94323952b777ff1bbb0a2 is 50, key is test_row_0/B:col10/1731033203002/Put/seqid=0 2024-11-08T02:33:24,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742349_1525 (size=12151) 2024-11-08T02:33:24,933 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/c8d537f21cf94323952b777ff1bbb0a2 2024-11-08T02:33:24,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-08T02:33:24,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/1cbe920d4b1b4cbdbcb11cc1f794569d is 50, key is test_row_0/C:col10/1731033203002/Put/seqid=0 2024-11-08T02:33:24,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742350_1526 (size=12151) 2024-11-08T02:33:25,058 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T02:33:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:25,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033265159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033265160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033265160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033265161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033265262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033265263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033265263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033265264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/1cbe920d4b1b4cbdbcb11cc1f794569d 2024-11-08T02:33:25,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/3ee432405bda4f9ab005caae0578417e as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3ee432405bda4f9ab005caae0578417e 2024-11-08T02:33:25,349 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3ee432405bda4f9ab005caae0578417e, entries=150, sequenceid=211, filesize=30.4 K 2024-11-08T02:33:25,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/c8d537f21cf94323952b777ff1bbb0a2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8d537f21cf94323952b777ff1bbb0a2 2024-11-08T02:33:25,352 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8d537f21cf94323952b777ff1bbb0a2, entries=150, sequenceid=211, filesize=11.9 K 2024-11-08T02:33:25,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/1cbe920d4b1b4cbdbcb11cc1f794569d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1cbe920d4b1b4cbdbcb11cc1f794569d 2024-11-08T02:33:25,356 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1cbe920d4b1b4cbdbcb11cc1f794569d, entries=150, sequenceid=211, filesize=11.9 K 2024-11-08T02:33:25,356 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 004f6c88876d7274b3d444aacc1fbc17 in 871ms, sequenceid=211, compaction requested=false 2024-11-08T02:33:25,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:25,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:25,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=162}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=162 2024-11-08T02:33:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=162 2024-11-08T02:33:25,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=162, resume processing ppid=161 2024-11-08T02:33:25,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=162, ppid=161, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0240 sec 2024-11-08T02:33:25,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=161, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=161, table=TestAcidGuarantees in 1.0260 sec 2024-11-08T02:33:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-08T02:33:25,436 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 161 completed 2024-11-08T02:33:25,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=163, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees 2024-11-08T02:33:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-08T02:33:25,438 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=163, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:25,438 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=163, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:25,438 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:25,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T02:33:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:25,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:25,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086cd1a870bc4041d7b8a5972d0bc44180_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033205466/Put/seqid=0 2024-11-08T02:33:25,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742351_1527 (size=14794) 2024-11-08T02:33:25,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033265477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033265480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033265481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033265481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-08T02:33:25,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033265582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033265584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033265584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033265585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,589 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=164 2024-11-08T02:33:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:25,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:25,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] handler.RSProcedureHandler(58): pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=164 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-08T02:33:25,741 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=164 2024-11-08T02:33:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:25,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] handler.RSProcedureHandler(58): pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=164 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033265784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033265787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033265788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033265788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,877 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:25,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411086cd1a870bc4041d7b8a5972d0bc44180_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086cd1a870bc4041d7b8a5972d0bc44180_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:25,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/bf141b67034f4ccfbab9897afa6389b8, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:25,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/bf141b67034f4ccfbab9897afa6389b8 is 175, key is test_row_0/A:col10/1731033205466/Put/seqid=0 2024-11-08T02:33:25,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742352_1528 (size=39749) 2024-11-08T02:33:25,893 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:25,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=164 2024-11-08T02:33:25,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:25,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:25,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:25,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] handler.RSProcedureHandler(58): pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=164 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-08T02:33:26,045 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=164 2024-11-08T02:33:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] handler.RSProcedureHandler(58): pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=164 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:26,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033266087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033266091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033266091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033266092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,197 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=164 2024-11-08T02:33:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] handler.RSProcedureHandler(58): pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-2 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=164 java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4114): Remote procedure failed, pid=164 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:26,284 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/bf141b67034f4ccfbab9897afa6389b8 2024-11-08T02:33:26,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/08e6e8cadb314da3b9ab542aac6a8c7a is 50, key is test_row_0/B:col10/1731033205466/Put/seqid=0 2024-11-08T02:33:26,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742353_1529 (size=12151) 2024-11-08T02:33:26,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/08e6e8cadb314da3b9ab542aac6a8c7a 2024-11-08T02:33:26,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/00fa5af529d7487fbeba87e25ebdf8c4 is 50, key is test_row_0/C:col10/1731033205466/Put/seqid=0 2024-11-08T02:33:26,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742354_1530 (size=12151) 2024-11-08T02:33:26,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/00fa5af529d7487fbeba87e25ebdf8c4 2024-11-08T02:33:26,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/bf141b67034f4ccfbab9897afa6389b8 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/bf141b67034f4ccfbab9897afa6389b8 2024-11-08T02:33:26,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/bf141b67034f4ccfbab9897afa6389b8, entries=200, sequenceid=238, filesize=38.8 K 2024-11-08T02:33:26,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/08e6e8cadb314da3b9ab542aac6a8c7a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/08e6e8cadb314da3b9ab542aac6a8c7a 2024-11-08T02:33:26,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/08e6e8cadb314da3b9ab542aac6a8c7a, entries=150, sequenceid=238, filesize=11.9 K 2024-11-08T02:33:26,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/00fa5af529d7487fbeba87e25ebdf8c4 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/00fa5af529d7487fbeba87e25ebdf8c4 2024-11-08T02:33:26,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/00fa5af529d7487fbeba87e25ebdf8c4, entries=150, sequenceid=238, filesize=11.9 K 2024-11-08T02:33:26,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 004f6c88876d7274b3d444aacc1fbc17 in 857ms, sequenceid=238, compaction requested=true 2024-11-08T02:33:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:33:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:26,325 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:33:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:26,325 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:33:26,325 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:26,325 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:26,325 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/A is initiating minor compaction (all files) 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/B is initiating minor compaction (all files) 2024-11-08T02:33:26,326 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/B in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,326 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/A in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,326 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/969b1d93ff294d60a5b839bf5ad3053c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8d537f21cf94323952b777ff1bbb0a2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/08e6e8cadb314da3b9ab542aac6a8c7a] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=36.0 K 2024-11-08T02:33:26,326 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/7bd8d230b4794e50a2d360b4f7fb01ed, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3ee432405bda4f9ab005caae0578417e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/bf141b67034f4ccfbab9897afa6389b8] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=100.0 K 2024-11-08T02:33:26,326 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/7bd8d230b4794e50a2d360b4f7fb01ed, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3ee432405bda4f9ab005caae0578417e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/bf141b67034f4ccfbab9897afa6389b8] 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 969b1d93ff294d60a5b839bf5ad3053c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731033202381 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7bd8d230b4794e50a2d360b4f7fb01ed, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731033202381 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3ee432405bda4f9ab005caae0578417e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731033203000 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c8d537f21cf94323952b777ff1bbb0a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731033203000 2024-11-08T02:33:26,326 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting bf141b67034f4ccfbab9897afa6389b8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731033205158 2024-11-08T02:33:26,327 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 08e6e8cadb314da3b9ab542aac6a8c7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731033205158 2024-11-08T02:33:26,333 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:26,333 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/a57a668347cb45c68318ca800a390934 is 50, key is test_row_0/B:col10/1731033205466/Put/seqid=0 2024-11-08T02:33:26,336 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:26,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742355_1531 (size=12697) 2024-11-08T02:33:26,349 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=164 2024-11-08T02:33:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,350 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:33:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:26,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:26,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:26,355 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108ad78c83eb81c49969d3d147d41ea1774_004f6c88876d7274b3d444aacc1fbc17 store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:26,357 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108ad78c83eb81c49969d3d147d41ea1774_004f6c88876d7274b3d444aacc1fbc17, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:26,357 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ad78c83eb81c49969d3d147d41ea1774_004f6c88876d7274b3d444aacc1fbc17 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:26,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110875caafbee8024850a56d60ca0071bc2b_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033205480/Put/seqid=0 2024-11-08T02:33:26,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742356_1532 (size=4469) 2024-11-08T02:33:26,372 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#A#compaction#445 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:26,373 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d35cc0b8dcdf45b8843072fd62569536 is 175, key is test_row_0/A:col10/1731033205466/Put/seqid=0 2024-11-08T02:33:26,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742357_1533 (size=12304) 2024-11-08T02:33:26,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742358_1534 (size=31651) 2024-11-08T02:33:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:26,395 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110875caafbee8024850a56d60ca0071bc2b_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110875caafbee8024850a56d60ca0071bc2b_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:26,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4865a51dd3bd4e83a9e10f997bb04e98, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:26,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4865a51dd3bd4e83a9e10f997bb04e98 is 175, key is test_row_0/A:col10/1731033205480/Put/seqid=0 2024-11-08T02:33:26,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742359_1535 (size=31105) 2024-11-08T02:33:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-08T02:33:26,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:26,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:26,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033266615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033266616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033266617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033266618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033266719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033266719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033266720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033266722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,752 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/a57a668347cb45c68318ca800a390934 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a57a668347cb45c68318ca800a390934 2024-11-08T02:33:26,755 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/B of 004f6c88876d7274b3d444aacc1fbc17 into a57a668347cb45c68318ca800a390934(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:26,755 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:26,755 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/B, priority=13, startTime=1731033206325; duration=0sec 2024-11-08T02:33:26,755 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:26,755 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:B 2024-11-08T02:33:26,755 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:26,756 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:26,756 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/C is initiating minor compaction (all files) 2024-11-08T02:33:26,756 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/C in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:26,756 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b204e627bf9642c8baa3adbbb27df672, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1cbe920d4b1b4cbdbcb11cc1f794569d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/00fa5af529d7487fbeba87e25ebdf8c4] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=36.0 K 2024-11-08T02:33:26,756 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting b204e627bf9642c8baa3adbbb27df672, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731033202381 2024-11-08T02:33:26,757 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1cbe920d4b1b4cbdbcb11cc1f794569d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731033203000 2024-11-08T02:33:26,757 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 00fa5af529d7487fbeba87e25ebdf8c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731033205158 2024-11-08T02:33:26,762 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:26,762 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/842c8882d496463aa6c4c4e2543468ca is 50, key is test_row_0/C:col10/1731033205466/Put/seqid=0 2024-11-08T02:33:26,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742360_1536 (size=12697) 2024-11-08T02:33:26,798 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/d35cc0b8dcdf45b8843072fd62569536 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d35cc0b8dcdf45b8843072fd62569536 2024-11-08T02:33:26,799 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4865a51dd3bd4e83a9e10f997bb04e98 2024-11-08T02:33:26,802 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/A of 004f6c88876d7274b3d444aacc1fbc17 into d35cc0b8dcdf45b8843072fd62569536(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:26,802 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:26,802 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/A, priority=13, startTime=1731033206325; duration=0sec 2024-11-08T02:33:26,802 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:26,802 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:A 2024-11-08T02:33:26,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/38787579e2b146ed890b84dc45a5c058 is 50, key is test_row_0/B:col10/1731033205480/Put/seqid=0 2024-11-08T02:33:26,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742361_1537 (size=12151) 2024-11-08T02:33:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033266922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033266923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033266923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033266925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,169 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/842c8882d496463aa6c4c4e2543468ca as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/842c8882d496463aa6c4c4e2543468ca 2024-11-08T02:33:27,172 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/C of 004f6c88876d7274b3d444aacc1fbc17 into 842c8882d496463aa6c4c4e2543468ca(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:27,172 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:27,172 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/C, priority=13, startTime=1731033206325; duration=0sec 2024-11-08T02:33:27,172 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:27,172 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:C 2024-11-08T02:33:27,207 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/38787579e2b146ed890b84dc45a5c058 2024-11-08T02:33:27,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/8f22045170854458a7c9e6d51f868855 is 50, key is test_row_0/C:col10/1731033205480/Put/seqid=0 2024-11-08T02:33:27,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742362_1538 (size=12151) 2024-11-08T02:33:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033267226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033267227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033267227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033267228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-08T02:33:27,616 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/8f22045170854458a7c9e6d51f868855 2024-11-08T02:33:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4865a51dd3bd4e83a9e10f997bb04e98 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4865a51dd3bd4e83a9e10f997bb04e98 2024-11-08T02:33:27,621 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4865a51dd3bd4e83a9e10f997bb04e98, entries=150, sequenceid=248, filesize=30.4 K 2024-11-08T02:33:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/38787579e2b146ed890b84dc45a5c058 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/38787579e2b146ed890b84dc45a5c058 2024-11-08T02:33:27,625 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/38787579e2b146ed890b84dc45a5c058, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T02:33:27,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/8f22045170854458a7c9e6d51f868855 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8f22045170854458a7c9e6d51f868855 2024-11-08T02:33:27,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8f22045170854458a7c9e6d51f868855, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T02:33:27,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 004f6c88876d7274b3d444aacc1fbc17 in 1278ms, sequenceid=248, compaction requested=false 2024-11-08T02:33:27,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:27,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:27,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-0 {event_type=RS_FLUSH_REGIONS, pid=164}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=164 2024-11-08T02:33:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=164 2024-11-08T02:33:27,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=164, resume processing ppid=163 2024-11-08T02:33:27,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=164, ppid=163, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1920 sec 2024-11-08T02:33:27,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=163, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=163, table=TestAcidGuarantees in 2.1940 sec 2024-11-08T02:33:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:27,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T02:33:27,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ca01a1ae0f0c41198f098e490a5748c0_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033207731/Put/seqid=0 2024-11-08T02:33:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033267735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033267736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033267738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033267738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742363_1539 (size=12454) 2024-11-08T02:33:27,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033267839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033267839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033267841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:27,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033267841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033268041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033268042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033268044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033268044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,145 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:28,147 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108ca01a1ae0f0c41198f098e490a5748c0_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ca01a1ae0f0c41198f098e490a5748c0_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:28,148 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/a82283477a9b48c18fb1bc1c483f0c9b, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:28,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/a82283477a9b48c18fb1bc1c483f0c9b is 175, key is test_row_0/A:col10/1731033207731/Put/seqid=0 2024-11-08T02:33:28,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742364_1540 (size=31255) 2024-11-08T02:33:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033268345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033268345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033268346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033268346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/a82283477a9b48c18fb1bc1c483f0c9b 2024-11-08T02:33:28,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/f129b97686bf4c60bcf31df54abd1ab5 is 50, key is test_row_0/B:col10/1731033207731/Put/seqid=0 2024-11-08T02:33:28,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742365_1541 (size=12301) 2024-11-08T02:33:28,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033268848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033268849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033268850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033268851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/f129b97686bf4c60bcf31df54abd1ab5 2024-11-08T02:33:28,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/dfa280e456de482c947e597a0943108b is 50, key is test_row_0/C:col10/1731033207731/Put/seqid=0 2024-11-08T02:33:28,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742366_1542 (size=12301) 2024-11-08T02:33:28,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/dfa280e456de482c947e597a0943108b 2024-11-08T02:33:28,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/a82283477a9b48c18fb1bc1c483f0c9b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/a82283477a9b48c18fb1bc1c483f0c9b 2024-11-08T02:33:28,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/a82283477a9b48c18fb1bc1c483f0c9b, entries=150, sequenceid=278, filesize=30.5 K 2024-11-08T02:33:28,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/f129b97686bf4c60bcf31df54abd1ab5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f129b97686bf4c60bcf31df54abd1ab5 2024-11-08T02:33:28,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f129b97686bf4c60bcf31df54abd1ab5, entries=150, sequenceid=278, filesize=12.0 K 2024-11-08T02:33:28,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/dfa280e456de482c947e597a0943108b as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/dfa280e456de482c947e597a0943108b 2024-11-08T02:33:28,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/dfa280e456de482c947e597a0943108b, entries=150, sequenceid=278, filesize=12.0 K 2024-11-08T02:33:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52390 deadline: 1731033268980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:28,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 004f6c88876d7274b3d444aacc1fbc17 in 1249ms, sequenceid=278, compaction requested=true 2024-11-08T02:33:28,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:28,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:33:28,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:28,981 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:28,981 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:33:28,982 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:28,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:28,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:33:28,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:28,982 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94011 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:28,982 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:28,982 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/A is initiating minor compaction (all files) 2024-11-08T02:33:28,982 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/B is initiating minor compaction (all files) 2024-11-08T02:33:28,982 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/A in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:28,982 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/B in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:28,982 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d35cc0b8dcdf45b8843072fd62569536, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4865a51dd3bd4e83a9e10f997bb04e98, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/a82283477a9b48c18fb1bc1c483f0c9b] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=91.8 K 2024-11-08T02:33:28,982 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a57a668347cb45c68318ca800a390934, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/38787579e2b146ed890b84dc45a5c058, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f129b97686bf4c60bcf31df54abd1ab5] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=36.3 K 2024-11-08T02:33:28,982 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:28,982 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d35cc0b8dcdf45b8843072fd62569536, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4865a51dd3bd4e83a9e10f997bb04e98, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/a82283477a9b48c18fb1bc1c483f0c9b] 2024-11-08T02:33:28,983 DEBUG [Thread-2106 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18183 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., hostname=ecdbff65922a,40577,1731033026611, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T02:33:28,983 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting d35cc0b8dcdf45b8843072fd62569536, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731033205158 2024-11-08T02:33:28,983 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a57a668347cb45c68318ca800a390934, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731033205158 2024-11-08T02:33:28,983 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 38787579e2b146ed890b84dc45a5c058, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731033205476 2024-11-08T02:33:28,983 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4865a51dd3bd4e83a9e10f997bb04e98, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731033205476 2024-11-08T02:33:28,983 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting f129b97686bf4c60bcf31df54abd1ab5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731033206614 2024-11-08T02:33:28,983 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting a82283477a9b48c18fb1bc1c483f0c9b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731033206614 2024-11-08T02:33:28,987 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:28,989 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108f0707524044b4088aa6b17e09224624a_004f6c88876d7274b3d444aacc1fbc17 store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:28,989 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#B#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:28,989 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/c8a14449803d410d94d732e09054a833 is 50, key is test_row_0/B:col10/1731033207731/Put/seqid=0 2024-11-08T02:33:28,990 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108f0707524044b4088aa6b17e09224624a_004f6c88876d7274b3d444aacc1fbc17, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:28,990 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108f0707524044b4088aa6b17e09224624a_004f6c88876d7274b3d444aacc1fbc17 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:28,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742368_1544 (size=4469) 2024-11-08T02:33:28,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742367_1543 (size=12949) 2024-11-08T02:33:29,394 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#A#compaction#454 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:29,394 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/53e59712d10541f29887dc3124b3a2f9 is 175, key is test_row_0/A:col10/1731033207731/Put/seqid=0 2024-11-08T02:33:29,397 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/c8a14449803d410d94d732e09054a833 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8a14449803d410d94d732e09054a833 2024-11-08T02:33:29,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742369_1545 (size=31903) 2024-11-08T02:33:29,400 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/53e59712d10541f29887dc3124b3a2f9 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/53e59712d10541f29887dc3124b3a2f9 2024-11-08T02:33:29,401 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/B of 004f6c88876d7274b3d444aacc1fbc17 into c8a14449803d410d94d732e09054a833(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:29,401 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:29,401 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/B, priority=13, startTime=1731033208981; duration=0sec 2024-11-08T02:33:29,401 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:29,401 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:B 2024-11-08T02:33:29,401 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:29,402 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:29,402 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/C is initiating minor compaction (all files) 2024-11-08T02:33:29,402 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/C in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:29,402 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/842c8882d496463aa6c4c4e2543468ca, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8f22045170854458a7c9e6d51f868855, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/dfa280e456de482c947e597a0943108b] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=36.3 K 2024-11-08T02:33:29,402 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 842c8882d496463aa6c4c4e2543468ca, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731033205158 2024-11-08T02:33:29,402 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 8f22045170854458a7c9e6d51f868855, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731033205476 2024-11-08T02:33:29,402 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting dfa280e456de482c947e597a0943108b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731033206614 2024-11-08T02:33:29,404 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/A of 004f6c88876d7274b3d444aacc1fbc17 into 53e59712d10541f29887dc3124b3a2f9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:29,404 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:29,404 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/A, priority=13, startTime=1731033208981; duration=0sec 2024-11-08T02:33:29,404 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:29,404 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:A 2024-11-08T02:33:29,407 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#C#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:29,407 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/eea6cd6c45eb4737b3a3b8fb3b952e45 is 50, key is test_row_0/C:col10/1731033207731/Put/seqid=0 2024-11-08T02:33:29,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742370_1546 (size=12949) 2024-11-08T02:33:29,414 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/eea6cd6c45eb4737b3a3b8fb3b952e45 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/eea6cd6c45eb4737b3a3b8fb3b952e45 2024-11-08T02:33:29,417 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/C of 004f6c88876d7274b3d444aacc1fbc17 into eea6cd6c45eb4737b3a3b8fb3b952e45(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:29,417 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:29,417 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/C, priority=13, startTime=1731033208982; duration=0sec 2024-11-08T02:33:29,417 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:29,417 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:C 2024-11-08T02:33:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-08T02:33:29,541 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 163 completed 2024-11-08T02:33:29,542 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T02:33:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=165, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees 2024-11-08T02:33:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-08T02:33:29,543 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=165, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T02:33:29,544 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=165, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T02:33:29,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T02:33:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-08T02:33:29,695 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40577 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=166 2024-11-08T02:33:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:29,695 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T02:33:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:29,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:29,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:29,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:29,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:29,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:29,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108573099d788604e6aae1ba0ef9c705ebb_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033207737/Put/seqid=0 2024-11-08T02:33:29,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742371_1547 (size=12454) 2024-11-08T02:33:29,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:29,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108573099d788604e6aae1ba0ef9c705ebb_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108573099d788604e6aae1ba0ef9c705ebb_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:29,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/68cfd3840c4446baa18d11b37b05d58f, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:29,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/68cfd3840c4446baa18d11b37b05d58f is 175, key is test_row_0/A:col10/1731033207737/Put/seqid=0 2024-11-08T02:33:29,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742372_1548 (size=31255) 2024-11-08T02:33:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-08T02:33:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:29,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. as already flushing 2024-11-08T02:33:29,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033269869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033269870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033269871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033269871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033269973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033269974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033269974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:29,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033269975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,115 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/68cfd3840c4446baa18d11b37b05d58f 2024-11-08T02:33:30,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/1bcb34dc224d410d855870fe71b0aa81 is 50, key is test_row_0/B:col10/1731033207737/Put/seqid=0 2024-11-08T02:33:30,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-08T02:33:30,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742373_1549 (size=12301) 2024-11-08T02:33:30,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033270177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033270177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033270178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033270178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52346 deadline: 1731033270481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52318 deadline: 1731033270481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52350 deadline: 1731033270481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-08T02:33:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40577 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52362 deadline: 1731033270482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:30,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/1bcb34dc224d410d855870fe71b0aa81 2024-11-08T02:33:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/a5e97dccbaf14d97b94e28b5235082b2 is 50, key is test_row_0/C:col10/1731033207737/Put/seqid=0 2024-11-08T02:33:30,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742374_1550 (size=12301) 2024-11-08T02:33:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-08T02:33:30,803 DEBUG [Thread-2117 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a3b66d3 to 127.0.0.1:56877 2024-11-08T02:33:30,803 DEBUG [Thread-2117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,804 DEBUG [Thread-2115 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x150e08ed to 127.0.0.1:56877 2024-11-08T02:33:30,804 DEBUG [Thread-2115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,804 DEBUG [Thread-2119 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5cfdf76c to 127.0.0.1:56877 2024-11-08T02:33:30,804 DEBUG [Thread-2119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,805 DEBUG [Thread-2123 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50bf224f to 127.0.0.1:56877 2024-11-08T02:33:30,805 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,805 DEBUG [Thread-2121 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68c2838a to 127.0.0.1:56877 2024-11-08T02:33:30,805 DEBUG [Thread-2121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,954 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/a5e97dccbaf14d97b94e28b5235082b2 2024-11-08T02:33:30,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/68cfd3840c4446baa18d11b37b05d58f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/68cfd3840c4446baa18d11b37b05d58f 2024-11-08T02:33:30,970 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/68cfd3840c4446baa18d11b37b05d58f, entries=150, sequenceid=291, filesize=30.5 K 2024-11-08T02:33:30,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/1bcb34dc224d410d855870fe71b0aa81 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1bcb34dc224d410d855870fe71b0aa81 2024-11-08T02:33:30,974 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1bcb34dc224d410d855870fe71b0aa81, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T02:33:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/a5e97dccbaf14d97b94e28b5235082b2 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/a5e97dccbaf14d97b94e28b5235082b2 2024-11-08T02:33:30,978 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/a5e97dccbaf14d97b94e28b5235082b2, entries=150, sequenceid=291, filesize=12.0 K 2024-11-08T02:33:30,978 INFO [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 004f6c88876d7274b3d444aacc1fbc17 in 1283ms, sequenceid=291, compaction requested=false 2024-11-08T02:33:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:30,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ecdbff65922a:0-1 {event_type=RS_FLUSH_REGIONS, pid=166}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=166 2024-11-08T02:33:30,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster(4106): Remote procedure done, pid=166 2024-11-08T02:33:30,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=166, resume processing ppid=165 2024-11-08T02:33:30,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=166, ppid=165, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4350 sec 2024-11-08T02:33:30,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=165, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=165, table=TestAcidGuarantees in 1.4390 sec 2024-11-08T02:33:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40577 {}] regionserver.HRegion(8581): Flush requested on 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:30,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T02:33:30,986 DEBUG [Thread-2108 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x11030ef5 to 127.0.0.1:56877 2024-11-08T02:33:30,986 DEBUG [Thread-2110 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69abefea to 127.0.0.1:56877 2024-11-08T02:33:30,986 DEBUG [Thread-2110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,986 DEBUG [Thread-2108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,987 DEBUG [Thread-2104 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x05bc9c3e to 127.0.0.1:56877 2024-11-08T02:33:30,987 DEBUG [Thread-2104 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:30,987 DEBUG [Thread-2112 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e757135 to 127.0.0.1:56877 2024-11-08T02:33:30,988 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:30,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:30,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:30,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:30,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:30,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:30,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081b122e423938491ca06ccd9fba8c96b4_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033210985/Put/seqid=0 2024-11-08T02:33:30,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742375_1551 (size=12454) 2024-11-08T02:33:31,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:31,409 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081b122e423938491ca06ccd9fba8c96b4_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081b122e423938491ca06ccd9fba8c96b4_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:31,410 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4b4d0c172e6049c2a70209aae7a769d7, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:31,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4b4d0c172e6049c2a70209aae7a769d7 is 175, key is test_row_0/A:col10/1731033210985/Put/seqid=0 2024-11-08T02:33:31,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742376_1552 (size=31255) 2024-11-08T02:33:31,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-08T02:33:31,648 INFO [Thread-2114 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 165 completed 2024-11-08T02:33:31,816 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4b4d0c172e6049c2a70209aae7a769d7 2024-11-08T02:33:31,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/d63e5e5d95874c7ab81948a19a1c5568 is 50, key is test_row_0/B:col10/1731033210985/Put/seqid=0 2024-11-08T02:33:31,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742377_1553 (size=12301) 2024-11-08T02:33:32,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/d63e5e5d95874c7ab81948a19a1c5568 2024-11-08T02:33:32,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/e117c44e530f42d99250f3a8dc3e8bac is 50, key is test_row_0/C:col10/1731033210985/Put/seqid=0 2024-11-08T02:33:32,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742378_1554 (size=12301) 2024-11-08T02:33:32,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/e117c44e530f42d99250f3a8dc3e8bac 2024-11-08T02:33:32,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/4b4d0c172e6049c2a70209aae7a769d7 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4b4d0c172e6049c2a70209aae7a769d7 2024-11-08T02:33:32,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4b4d0c172e6049c2a70209aae7a769d7, entries=150, sequenceid=321, filesize=30.5 K 2024-11-08T02:33:32,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/d63e5e5d95874c7ab81948a19a1c5568 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d63e5e5d95874c7ab81948a19a1c5568 2024-11-08T02:33:32,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d63e5e5d95874c7ab81948a19a1c5568, entries=150, sequenceid=321, filesize=12.0 K 2024-11-08T02:33:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/e117c44e530f42d99250f3a8dc3e8bac as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/e117c44e530f42d99250f3a8dc3e8bac 2024-11-08T02:33:32,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/e117c44e530f42d99250f3a8dc3e8bac, entries=150, sequenceid=321, filesize=12.0 K 2024-11-08T02:33:32,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=0 B/0 for 004f6c88876d7274b3d444aacc1fbc17 in 1687ms, sequenceid=321, compaction requested=true 2024-11-08T02:33:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T02:33:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T02:33:32,674 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 004f6c88876d7274b3d444aacc1fbc17:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T02:33:32,674 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:32,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:32,675 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:32,675 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:32,675 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/A is initiating minor compaction (all files) 2024-11-08T02:33:32,675 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/B is initiating minor compaction (all files) 2024-11-08T02:33:32,675 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/A in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:32,675 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/B in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:32,675 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/53e59712d10541f29887dc3124b3a2f9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/68cfd3840c4446baa18d11b37b05d58f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4b4d0c172e6049c2a70209aae7a769d7] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=92.2 K 2024-11-08T02:33:32,675 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8a14449803d410d94d732e09054a833, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1bcb34dc224d410d855870fe71b0aa81, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d63e5e5d95874c7ab81948a19a1c5568] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=36.7 K 2024-11-08T02:33:32,675 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:32,675 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. files: [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/53e59712d10541f29887dc3124b3a2f9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/68cfd3840c4446baa18d11b37b05d58f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4b4d0c172e6049c2a70209aae7a769d7] 2024-11-08T02:33:32,675 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting c8a14449803d410d94d732e09054a833, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731033206614 2024-11-08T02:33:32,675 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 53e59712d10541f29887dc3124b3a2f9, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731033206614 2024-11-08T02:33:32,676 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting 1bcb34dc224d410d855870fe71b0aa81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033207735 2024-11-08T02:33:32,676 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 68cfd3840c4446baa18d11b37b05d58f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033207735 2024-11-08T02:33:32,676 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting d63e5e5d95874c7ab81948a19a1c5568, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731033210985 2024-11-08T02:33:32,676 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4b4d0c172e6049c2a70209aae7a769d7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731033210985 2024-11-08T02:33:32,683 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:32,684 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#B#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:32,684 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/5687c48169fb4a47845c3a3bf538c7f3 is 50, key is test_row_0/B:col10/1731033210985/Put/seqid=0 2024-11-08T02:33:32,684 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411087e045bdbaf8b47338427b51428e0a414_004f6c88876d7274b3d444aacc1fbc17 store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:32,687 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411087e045bdbaf8b47338427b51428e0a414_004f6c88876d7274b3d444aacc1fbc17, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:32,687 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087e045bdbaf8b47338427b51428e0a414_004f6c88876d7274b3d444aacc1fbc17 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:32,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742379_1555 (size=13051) 2024-11-08T02:33:32,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742380_1556 (size=4469) 2024-11-08T02:33:33,094 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#A#compaction#463 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:33,096 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/2fcb64f078a1404eb06fe1a3a059475a is 175, key is test_row_0/A:col10/1731033210985/Put/seqid=0 2024-11-08T02:33:33,099 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/5687c48169fb4a47845c3a3bf538c7f3 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/5687c48169fb4a47845c3a3bf538c7f3 2024-11-08T02:33:33,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742381_1557 (size=32005) 2024-11-08T02:33:33,103 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/B of 004f6c88876d7274b3d444aacc1fbc17 into 5687c48169fb4a47845c3a3bf538c7f3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:33,103 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:33,104 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/B, priority=13, startTime=1731033212674; duration=0sec 2024-11-08T02:33:33,104 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T02:33:33,104 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:B 2024-11-08T02:33:33,104 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T02:33:33,104 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T02:33:33,104 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1540): 004f6c88876d7274b3d444aacc1fbc17/C is initiating minor compaction (all files) 2024-11-08T02:33:33,105 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 004f6c88876d7274b3d444aacc1fbc17/C in TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:33,105 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/eea6cd6c45eb4737b3a3b8fb3b952e45, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/a5e97dccbaf14d97b94e28b5235082b2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/e117c44e530f42d99250f3a8dc3e8bac] into tmpdir=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp, totalSize=36.7 K 2024-11-08T02:33:33,105 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting eea6cd6c45eb4737b3a3b8fb3b952e45, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731033206614 2024-11-08T02:33:33,105 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting a5e97dccbaf14d97b94e28b5235082b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731033207735 2024-11-08T02:33:33,105 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] compactions.Compactor(224): Compacting e117c44e530f42d99250f3a8dc3e8bac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731033210985 2024-11-08T02:33:33,111 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 004f6c88876d7274b3d444aacc1fbc17#C#compaction#464 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T02:33:33,111 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/aea248f7004340edb45913856833a207 is 50, key is test_row_0/C:col10/1731033210985/Put/seqid=0 2024-11-08T02:33:33,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742382_1558 (size=13051) 2024-11-08T02:33:33,509 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/2fcb64f078a1404eb06fe1a3a059475a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2fcb64f078a1404eb06fe1a3a059475a 2024-11-08T02:33:33,513 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/A of 004f6c88876d7274b3d444aacc1fbc17 into 2fcb64f078a1404eb06fe1a3a059475a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:33,513 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:33,513 INFO [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/A, priority=13, startTime=1731033212674; duration=0sec 2024-11-08T02:33:33,514 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:33,514 DEBUG [RS:0;ecdbff65922a:40577-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:A 2024-11-08T02:33:33,518 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/aea248f7004340edb45913856833a207 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/aea248f7004340edb45913856833a207 2024-11-08T02:33:33,522 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 004f6c88876d7274b3d444aacc1fbc17/C of 004f6c88876d7274b3d444aacc1fbc17 into aea248f7004340edb45913856833a207(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T02:33:33,522 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:33,522 INFO [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17., storeName=004f6c88876d7274b3d444aacc1fbc17/C, priority=13, startTime=1731033212674; duration=0sec 2024-11-08T02:33:33,522 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T02:33:33,522 DEBUG [RS:0;ecdbff65922a:40577-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 004f6c88876d7274b3d444aacc1fbc17:C 2024-11-08T02:33:35,814 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46319,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:37,570 ERROR [LeaseRenewer:jenkins@localhost:46319 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46319,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T02:33:39,002 DEBUG [Thread-2106 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7181df3b to 127.0.0.1:56877 2024-11-08T02:33:39,002 DEBUG [Thread-2106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 1 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6839 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6669 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6559 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6817 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6678 2024-11-08T02:33:39,003 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T02:33:39,003 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T02:33:39,003 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2ac53e79 to 127.0.0.1:56877 2024-11-08T02:33:39,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:39,004 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-08T02:33:39,004 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T02:33:39,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=167, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:39,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-08T02:33:39,006 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033219006"}]},"ts":"1731033219006"} 2024-11-08T02:33:39,007 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T02:33:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-08T02:33:39,178 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T02:33:39,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T02:33:39,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, UNASSIGN}] 2024-11-08T02:33:39,185 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=169, ppid=168, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, UNASSIGN 2024-11-08T02:33:39,187 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=CLOSING, regionLocation=ecdbff65922a,40577,1731033026611 2024-11-08T02:33:39,189 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T02:33:39,189 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE; CloseRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611}] 2024-11-08T02:33:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-08T02:33:39,341 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to ecdbff65922a,40577,1731033026611 2024-11-08T02:33:39,342 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(124): Close 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:39,343 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-08T02:33:39,343 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1681): Closing 004f6c88876d7274b3d444aacc1fbc17, disabling compactions & flushes 2024-11-08T02:33:39,343 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:39,343 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:39,343 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. after waiting 0 ms 2024-11-08T02:33:39,343 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:39,343 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(2837): Flushing 004f6c88876d7274b3d444aacc1fbc17 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-08T02:33:39,344 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=A 2024-11-08T02:33:39,344 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:39,344 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=B 2024-11-08T02:33:39,344 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:39,344 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 004f6c88876d7274b3d444aacc1fbc17, store=C 2024-11-08T02:33:39,345 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-08T02:33:39,353 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108766cddd9da6942d2bfe01352d155801e_004f6c88876d7274b3d444aacc1fbc17 is 50, key is test_row_0/A:col10/1731033219002/Put/seqid=0 2024-11-08T02:33:39,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742383_1559 (size=7374) 2024-11-08T02:33:39,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-08T02:33:39,759 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T02:33:39,768 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108766cddd9da6942d2bfe01352d155801e_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108766cddd9da6942d2bfe01352d155801e_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:39,769 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/033fef6a32474887b32f7e1db8d3a695, store: [table=TestAcidGuarantees family=A region=004f6c88876d7274b3d444aacc1fbc17] 2024-11-08T02:33:39,770 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/033fef6a32474887b32f7e1db8d3a695 is 175, key is test_row_0/A:col10/1731033219002/Put/seqid=0 2024-11-08T02:33:39,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742384_1560 (size=13865) 2024-11-08T02:33:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-08T02:33:40,176 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/033fef6a32474887b32f7e1db8d3a695 2024-11-08T02:33:40,188 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b0c43820e1c546dbad913189b93c20b5 is 50, key is test_row_0/B:col10/1731033219002/Put/seqid=0 2024-11-08T02:33:40,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742385_1561 (size=7415) 2024-11-08T02:33:40,592 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b0c43820e1c546dbad913189b93c20b5 2024-11-08T02:33:40,604 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/24e86339f37c4dcc9c5144abf675f8ef is 50, key is test_row_0/C:col10/1731033219002/Put/seqid=0 2024-11-08T02:33:40,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742386_1562 (size=7415) 2024-11-08T02:33:41,011 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/24e86339f37c4dcc9c5144abf675f8ef 2024-11-08T02:33:41,020 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/A/033fef6a32474887b32f7e1db8d3a695 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/033fef6a32474887b32f7e1db8d3a695 2024-11-08T02:33:41,025 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/033fef6a32474887b32f7e1db8d3a695, entries=50, sequenceid=328, filesize=13.5 K 2024-11-08T02:33:41,026 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/B/b0c43820e1c546dbad913189b93c20b5 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b0c43820e1c546dbad913189b93c20b5 2024-11-08T02:33:41,030 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b0c43820e1c546dbad913189b93c20b5, entries=50, sequenceid=328, filesize=7.2 K 2024-11-08T02:33:41,031 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/.tmp/C/24e86339f37c4dcc9c5144abf675f8ef as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/24e86339f37c4dcc9c5144abf675f8ef 2024-11-08T02:33:41,034 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/24e86339f37c4dcc9c5144abf675f8ef, entries=50, sequenceid=328, filesize=7.2 K 2024-11-08T02:33:41,035 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(3040): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 004f6c88876d7274b3d444aacc1fbc17 in 1692ms, sequenceid=328, compaction requested=false 2024-11-08T02:33:41,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/e0037c20383c4215a5ca34e67e181389, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/49539e1815464d26b84461851a2d67bb, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/93a4bba71d5343d6b5cf9bcafb79fdb4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3875ab90f4d74f4a87cc79a88a8b67d3, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4da8ca0d2fc34f5baeede8da52d884cf, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d12348139d6a4664b64a5d349db0a17c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/713b93d475a44bd6a786ce562e0953a7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d78d972dfc9447c7879913d1ff262a32, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/b51da5dab1394c92a61f5aa8633adb10, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/fa69ddcc15eb4da8abe5efb45f1a3437, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2f8712da666c41f9866ed8fa96a8865f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3928c5f090b74538bb0e1d472cf146a8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/7bd8d230b4794e50a2d360b4f7fb01ed, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/f610108eec6d46e5822fba1c2f382905, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3ee432405bda4f9ab005caae0578417e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/bf141b67034f4ccfbab9897afa6389b8, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d35cc0b8dcdf45b8843072fd62569536, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4865a51dd3bd4e83a9e10f997bb04e98, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/53e59712d10541f29887dc3124b3a2f9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/a82283477a9b48c18fb1bc1c483f0c9b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/68cfd3840c4446baa18d11b37b05d58f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4b4d0c172e6049c2a70209aae7a769d7] to archive 2024-11-08T02:33:41,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:33:41,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/e0037c20383c4215a5ca34e67e181389 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/e0037c20383c4215a5ca34e67e181389 2024-11-08T02:33:41,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/49539e1815464d26b84461851a2d67bb to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/49539e1815464d26b84461851a2d67bb 2024-11-08T02:33:41,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/93a4bba71d5343d6b5cf9bcafb79fdb4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/93a4bba71d5343d6b5cf9bcafb79fdb4 2024-11-08T02:33:41,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3875ab90f4d74f4a87cc79a88a8b67d3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3875ab90f4d74f4a87cc79a88a8b67d3 2024-11-08T02:33:41,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4da8ca0d2fc34f5baeede8da52d884cf to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4da8ca0d2fc34f5baeede8da52d884cf 2024-11-08T02:33:41,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d12348139d6a4664b64a5d349db0a17c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d12348139d6a4664b64a5d349db0a17c 2024-11-08T02:33:41,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/713b93d475a44bd6a786ce562e0953a7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/713b93d475a44bd6a786ce562e0953a7 2024-11-08T02:33:41,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d78d972dfc9447c7879913d1ff262a32 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d78d972dfc9447c7879913d1ff262a32 2024-11-08T02:33:41,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/b51da5dab1394c92a61f5aa8633adb10 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/b51da5dab1394c92a61f5aa8633adb10 2024-11-08T02:33:41,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/fa69ddcc15eb4da8abe5efb45f1a3437 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/fa69ddcc15eb4da8abe5efb45f1a3437 2024-11-08T02:33:41,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2f8712da666c41f9866ed8fa96a8865f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2f8712da666c41f9866ed8fa96a8865f 2024-11-08T02:33:41,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3928c5f090b74538bb0e1d472cf146a8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3928c5f090b74538bb0e1d472cf146a8 2024-11-08T02:33:41,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/7bd8d230b4794e50a2d360b4f7fb01ed to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/7bd8d230b4794e50a2d360b4f7fb01ed 2024-11-08T02:33:41,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/f610108eec6d46e5822fba1c2f382905 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/f610108eec6d46e5822fba1c2f382905 2024-11-08T02:33:41,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3ee432405bda4f9ab005caae0578417e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/3ee432405bda4f9ab005caae0578417e 2024-11-08T02:33:41,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/bf141b67034f4ccfbab9897afa6389b8 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/bf141b67034f4ccfbab9897afa6389b8 2024-11-08T02:33:41,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d35cc0b8dcdf45b8843072fd62569536 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/d35cc0b8dcdf45b8843072fd62569536 2024-11-08T02:33:41,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4865a51dd3bd4e83a9e10f997bb04e98 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4865a51dd3bd4e83a9e10f997bb04e98 2024-11-08T02:33:41,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/53e59712d10541f29887dc3124b3a2f9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/53e59712d10541f29887dc3124b3a2f9 2024-11-08T02:33:41,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/a82283477a9b48c18fb1bc1c483f0c9b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/a82283477a9b48c18fb1bc1c483f0c9b 2024-11-08T02:33:41,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/68cfd3840c4446baa18d11b37b05d58f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/68cfd3840c4446baa18d11b37b05d58f 2024-11-08T02:33:41,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4b4d0c172e6049c2a70209aae7a769d7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/4b4d0c172e6049c2a70209aae7a769d7 2024-11-08T02:33:41,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/8145b82ae4174af189bdd570f38c55d5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/70959e2659b54717a0e28617798d4f96, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f9a6b3c4ea1b4bac90d5e77f527d3a99, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1fac69e1693348bab64691a227e5a773, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b39d1ae771814726a9161ef847091e3d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a5c3a1a27ede4d21af6bf8620f4b8188, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d8feea37126643f4bb6a9506289fbbcc, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/617649466c04435b8012de57bcf88e80, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/7ba0ad33817641398bb299efb068cb25, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/cd7afcef66ee4255aa086b008bade5c4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/ccef4cbaa58541d28961fc39cdad85f7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/493802baa7cf446cbcc32d02c0b46a0d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/969b1d93ff294d60a5b839bf5ad3053c, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b9b8cd4834d845acb74d581d4bd290d9, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8d537f21cf94323952b777ff1bbb0a2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a57a668347cb45c68318ca800a390934, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/08e6e8cadb314da3b9ab542aac6a8c7a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/38787579e2b146ed890b84dc45a5c058, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8a14449803d410d94d732e09054a833, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f129b97686bf4c60bcf31df54abd1ab5, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1bcb34dc224d410d855870fe71b0aa81, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d63e5e5d95874c7ab81948a19a1c5568] to archive 2024-11-08T02:33:41,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:33:41,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/8145b82ae4174af189bdd570f38c55d5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/8145b82ae4174af189bdd570f38c55d5 2024-11-08T02:33:41,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/70959e2659b54717a0e28617798d4f96 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/70959e2659b54717a0e28617798d4f96 2024-11-08T02:33:41,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f9a6b3c4ea1b4bac90d5e77f527d3a99 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f9a6b3c4ea1b4bac90d5e77f527d3a99 2024-11-08T02:33:41,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1fac69e1693348bab64691a227e5a773 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1fac69e1693348bab64691a227e5a773 2024-11-08T02:33:41,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b39d1ae771814726a9161ef847091e3d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b39d1ae771814726a9161ef847091e3d 2024-11-08T02:33:41,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a5c3a1a27ede4d21af6bf8620f4b8188 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a5c3a1a27ede4d21af6bf8620f4b8188 2024-11-08T02:33:41,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d8feea37126643f4bb6a9506289fbbcc to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d8feea37126643f4bb6a9506289fbbcc 2024-11-08T02:33:41,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/617649466c04435b8012de57bcf88e80 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/617649466c04435b8012de57bcf88e80 2024-11-08T02:33:41,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/7ba0ad33817641398bb299efb068cb25 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/7ba0ad33817641398bb299efb068cb25 2024-11-08T02:33:41,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/cd7afcef66ee4255aa086b008bade5c4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/cd7afcef66ee4255aa086b008bade5c4 2024-11-08T02:33:41,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/ccef4cbaa58541d28961fc39cdad85f7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/ccef4cbaa58541d28961fc39cdad85f7 2024-11-08T02:33:41,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/493802baa7cf446cbcc32d02c0b46a0d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/493802baa7cf446cbcc32d02c0b46a0d 2024-11-08T02:33:41,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/969b1d93ff294d60a5b839bf5ad3053c to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/969b1d93ff294d60a5b839bf5ad3053c 2024-11-08T02:33:41,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b9b8cd4834d845acb74d581d4bd290d9 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b9b8cd4834d845acb74d581d4bd290d9 2024-11-08T02:33:41,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8d537f21cf94323952b777ff1bbb0a2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8d537f21cf94323952b777ff1bbb0a2 2024-11-08T02:33:41,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a57a668347cb45c68318ca800a390934 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/a57a668347cb45c68318ca800a390934 2024-11-08T02:33:41,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/08e6e8cadb314da3b9ab542aac6a8c7a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/08e6e8cadb314da3b9ab542aac6a8c7a 2024-11-08T02:33:41,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/38787579e2b146ed890b84dc45a5c058 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/38787579e2b146ed890b84dc45a5c058 2024-11-08T02:33:41,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8a14449803d410d94d732e09054a833 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/c8a14449803d410d94d732e09054a833 2024-11-08T02:33:41,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f129b97686bf4c60bcf31df54abd1ab5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/f129b97686bf4c60bcf31df54abd1ab5 2024-11-08T02:33:41,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1bcb34dc224d410d855870fe71b0aa81 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/1bcb34dc224d410d855870fe71b0aa81 2024-11-08T02:33:41,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d63e5e5d95874c7ab81948a19a1c5568 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/d63e5e5d95874c7ab81948a19a1c5568 2024-11-08T02:33:41,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/73ac849cb1e84893bef595a076c4f210, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/371d0be258a54f1e84edac90b87373f2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/7121df4274724e3f8f7a4cc91ec3b6e2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/5a6a855283ec452bb78dbca6dc600a31, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1ab3b605982d4a13a8222fd566f568bd, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/32956c117ca648c690b78bb2fa936b86, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/76050f9ec91b4f2b81fa5916d215265f, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8499864ea2fa466e9497b90e928c6e2a, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/478f6ae73ba74b64b37948253ae59617, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/cfb1be0b55574a4793db8e4ba682a4df, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/0f30ab661f5548bcb2d8c013098bdca7, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b204e627bf9642c8baa3adbbb27df672, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/f188ecdae7ac44d0aea651cb89a4fe05, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1cbe920d4b1b4cbdbcb11cc1f794569d, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/842c8882d496463aa6c4c4e2543468ca, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/00fa5af529d7487fbeba87e25ebdf8c4, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8f22045170854458a7c9e6d51f868855, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/eea6cd6c45eb4737b3a3b8fb3b952e45, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/dfa280e456de482c947e597a0943108b, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/a5e97dccbaf14d97b94e28b5235082b2, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/e117c44e530f42d99250f3a8dc3e8bac] to archive 2024-11-08T02:33:41,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T02:33:41,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/73ac849cb1e84893bef595a076c4f210 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/73ac849cb1e84893bef595a076c4f210 2024-11-08T02:33:41,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/371d0be258a54f1e84edac90b87373f2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/371d0be258a54f1e84edac90b87373f2 2024-11-08T02:33:41,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/7121df4274724e3f8f7a4cc91ec3b6e2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/7121df4274724e3f8f7a4cc91ec3b6e2 2024-11-08T02:33:41,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/5a6a855283ec452bb78dbca6dc600a31 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/5a6a855283ec452bb78dbca6dc600a31 2024-11-08T02:33:41,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1ab3b605982d4a13a8222fd566f568bd to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1ab3b605982d4a13a8222fd566f568bd 2024-11-08T02:33:41,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/32956c117ca648c690b78bb2fa936b86 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/32956c117ca648c690b78bb2fa936b86 2024-11-08T02:33:41,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/76050f9ec91b4f2b81fa5916d215265f to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/76050f9ec91b4f2b81fa5916d215265f 2024-11-08T02:33:41,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8499864ea2fa466e9497b90e928c6e2a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8499864ea2fa466e9497b90e928c6e2a 2024-11-08T02:33:41,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/478f6ae73ba74b64b37948253ae59617 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/478f6ae73ba74b64b37948253ae59617 2024-11-08T02:33:41,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/cfb1be0b55574a4793db8e4ba682a4df to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/cfb1be0b55574a4793db8e4ba682a4df 2024-11-08T02:33:41,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b40e6621d2fd4ff3ae9bd5202e4b1f1e 2024-11-08T02:33:41,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/0f30ab661f5548bcb2d8c013098bdca7 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/0f30ab661f5548bcb2d8c013098bdca7 2024-11-08T02:33:41,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b204e627bf9642c8baa3adbbb27df672 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/b204e627bf9642c8baa3adbbb27df672 2024-11-08T02:33:41,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/f188ecdae7ac44d0aea651cb89a4fe05 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/f188ecdae7ac44d0aea651cb89a4fe05 2024-11-08T02:33:41,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1cbe920d4b1b4cbdbcb11cc1f794569d to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/1cbe920d4b1b4cbdbcb11cc1f794569d 2024-11-08T02:33:41,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/842c8882d496463aa6c4c4e2543468ca to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/842c8882d496463aa6c4c4e2543468ca 2024-11-08T02:33:41,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/00fa5af529d7487fbeba87e25ebdf8c4 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/00fa5af529d7487fbeba87e25ebdf8c4 2024-11-08T02:33:41,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8f22045170854458a7c9e6d51f868855 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/8f22045170854458a7c9e6d51f868855 2024-11-08T02:33:41,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/eea6cd6c45eb4737b3a3b8fb3b952e45 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/eea6cd6c45eb4737b3a3b8fb3b952e45 2024-11-08T02:33:41,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/dfa280e456de482c947e597a0943108b to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/dfa280e456de482c947e597a0943108b 2024-11-08T02:33:41,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/a5e97dccbaf14d97b94e28b5235082b2 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/a5e97dccbaf14d97b94e28b5235082b2 2024-11-08T02:33:41,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/e117c44e530f42d99250f3a8dc3e8bac to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/e117c44e530f42d99250f3a8dc3e8bac 2024-11-08T02:33:41,096 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/recovered.edits/331.seqid, newMaxSeqId=331, maxSeqId=4 2024-11-08T02:33:41,096 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17. 2024-11-08T02:33:41,096 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] regionserver.HRegion(1635): Region close journal for 004f6c88876d7274b3d444aacc1fbc17: 2024-11-08T02:33:41,097 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION, pid=170}] handler.UnassignRegionHandler(170): Closed 004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:41,097 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=169 updating hbase:meta row=004f6c88876d7274b3d444aacc1fbc17, regionState=CLOSED 2024-11-08T02:33:41,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=170, resume processing ppid=169 2024-11-08T02:33:41,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=170, ppid=169, state=SUCCESS; CloseRegionProcedure 004f6c88876d7274b3d444aacc1fbc17, server=ecdbff65922a,40577,1731033026611 in 1.9090 sec 2024-11-08T02:33:41,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=169, resume processing ppid=168 2024-11-08T02:33:41,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=169, ppid=168, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=004f6c88876d7274b3d444aacc1fbc17, UNASSIGN in 1.9160 sec 2024-11-08T02:33:41,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=168, resume processing ppid=167 2024-11-08T02:33:41,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=168, ppid=167, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9190 sec 2024-11-08T02:33:41,100 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731033221100"}]},"ts":"1731033221100"} 2024-11-08T02:33:41,101 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T02:33:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-08T02:33:41,144 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T02:33:41,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=167, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 2.1410 sec 2024-11-08T02:33:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-08T02:33:43,116 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 167 completed 2024-11-08T02:33:43,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T02:33:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] procedure2.ProcedureExecutor(1098): Stored pid=171, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:43,122 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=171, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:43,123 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=171, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-11-08T02:33:43,127 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,131 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C, FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/recovered.edits] 2024-11-08T02:33:43,135 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/033fef6a32474887b32f7e1db8d3a695 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/033fef6a32474887b32f7e1db8d3a695 2024-11-08T02:33:43,137 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2fcb64f078a1404eb06fe1a3a059475a to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/A/2fcb64f078a1404eb06fe1a3a059475a 2024-11-08T02:33:43,140 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/5687c48169fb4a47845c3a3bf538c7f3 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/5687c48169fb4a47845c3a3bf538c7f3 2024-11-08T02:33:43,142 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b0c43820e1c546dbad913189b93c20b5 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/B/b0c43820e1c546dbad913189b93c20b5 2024-11-08T02:33:43,144 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/24e86339f37c4dcc9c5144abf675f8ef to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/24e86339f37c4dcc9c5144abf675f8ef 2024-11-08T02:33:43,146 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/aea248f7004340edb45913856833a207 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/C/aea248f7004340edb45913856833a207 2024-11-08T02:33:43,150 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/recovered.edits/331.seqid to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17/recovered.edits/331.seqid 2024-11-08T02:33:43,150 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/default/TestAcidGuarantees/004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,151 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T02:33:43,151 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:33:43,152 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T02:33:43,157 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c38274c63134e3286342baa0555571d_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c38274c63134e3286342baa0555571d_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110813763a0388d54acea1f9f05308718935_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110813763a0388d54acea1f9f05308718935_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,160 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081b122e423938491ca06ccd9fba8c96b4_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081b122e423938491ca06ccd9fba8c96b4_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,160 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081e6d3b31f07a47dc88a1e7cb42b754c2_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081e6d3b31f07a47dc88a1e7cb42b754c2_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082eb613e8e99043d49bd179da1192c721_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082eb613e8e99043d49bd179da1192c721_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084bff7e216f35405aa830f7d6ab2f35d8_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084bff7e216f35405aa830f7d6ab2f35d8_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,163 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108573099d788604e6aae1ba0ef9c705ebb_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108573099d788604e6aae1ba0ef9c705ebb_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110863c02d13827f4604898801d450e41fee_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110863c02d13827f4604898801d450e41fee_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086cd1a870bc4041d7b8a5972d0bc44180_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411086cd1a870bc4041d7b8a5972d0bc44180_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,165 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110875caafbee8024850a56d60ca0071bc2b_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110875caafbee8024850a56d60ca0071bc2b_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,166 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108765bd0ad3adb4f16b840347c767cd68f_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108765bd0ad3adb4f16b840347c767cd68f_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,167 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108766cddd9da6942d2bfe01352d155801e_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108766cddd9da6942d2bfe01352d155801e_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,167 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110898b5a3e4ab164a22a1cef430030b062c_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110898b5a3e4ab164a22a1cef430030b062c_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,168 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ca01a1ae0f0c41198f098e490a5748c0_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108ca01a1ae0f0c41198f098e490a5748c0_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,169 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e15c603861894ceb9f05800357a644f2_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e15c603861894ceb9f05800357a644f2_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,170 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e6f09ed820664fe5a238eec6b6822b40_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e6f09ed820664fe5a238eec6b6822b40_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,170 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f6413811cc434425a8a8e7a04666f2d6_004f6c88876d7274b3d444aacc1fbc17 to hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108f6413811cc434425a8a8e7a04666f2d6_004f6c88876d7274b3d444aacc1fbc17 2024-11-08T02:33:43,171 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T02:33:43,173 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=171, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:43,175 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T02:33:43,176 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T02:33:43,177 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=171, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:43,177 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T02:33:43,177 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731033223177"}]},"ts":"9223372036854775807"} 2024-11-08T02:33:43,178 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-08T02:33:43,178 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 004f6c88876d7274b3d444aacc1fbc17, NAME => 'TestAcidGuarantees,,1731033187562.004f6c88876d7274b3d444aacc1fbc17.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T02:33:43,178 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T02:33:43,178 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731033223178"}]},"ts":"9223372036854775807"} 2024-11-08T02:33:43,179 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-08T02:33:43,221 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=171, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T02:33:43,222 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=171, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 103 msec 2024-11-08T02:33:43,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-11-08T02:33:43,226 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 171 completed 2024-11-08T02:33:43,239 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMobGetAtomicity Thread=235 (was 236), OpenFileDescriptor=446 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 270), ProcessCount=11 (was 11), AvailableMemoryMB=2083 (was 2094) 2024-11-08T02:33:43,239 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-08T02:33:43,240 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-08T02:33:43,240 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e541e88 to 127.0.0.1:56877 2024-11-08T02:33:43,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:43,240 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T02:33:43,240 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1610069381, stopped=false 2024-11-08T02:33:43,240 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=ecdbff65922a,40841,1731033025873 2024-11-08T02:33:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T02:33:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T02:33:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:33:43,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:33:43,244 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-08T02:33:43,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:33:43,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:43,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T02:33:43,245 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ecdbff65922a,40577,1731033026611' ***** 2024-11-08T02:33:43,246 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-08T02:33:43,246 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T02:33:43,246 INFO [RS:0;ecdbff65922a:40577 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T02:33:43,246 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-08T02:33:43,246 INFO [RS:0;ecdbff65922a:40577 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T02:33:43,246 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(3579): Received CLOSE for 934a9f648376cf09654e404de226250e 2024-11-08T02:33:43,247 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1224): stopping server ecdbff65922a,40577,1731033026611 2024-11-08T02:33:43,247 DEBUG [RS:0;ecdbff65922a:40577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:43,247 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T02:33:43,247 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T02:33:43,247 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T02:33:43,247 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-08T02:33:43,247 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 934a9f648376cf09654e404de226250e, disabling compactions & flushes 2024-11-08T02:33:43,247 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-08T02:33:43,247 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:33:43,247 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 934a9f648376cf09654e404de226250e=hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e.} 2024-11-08T02:33:43,247 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:33:43,247 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. after waiting 0 ms 2024-11-08T02:33:43,248 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:33:43,248 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-08T02:33:43,248 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 934a9f648376cf09654e404de226250e 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-08T02:33:43,248 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-08T02:33:43,248 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-08T02:33:43,248 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T02:33:43,248 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T02:33:43,248 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=20.55 KB heapSize=35.87 KB 2024-11-08T02:33:43,251 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 934a9f648376cf09654e404de226250e 2024-11-08T02:33:43,265 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e/.tmp/info/01ff3865001248ffb41431fe7e8d9a61 is 45, key is default/info:d/1731033032379/Put/seqid=0 2024-11-08T02:33:43,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742387_1563 (size=5037) 2024-11-08T02:33:43,272 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/info/8f8a0e5bc87f4536847e5ba3a50faa0f is 143, key is hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e./info:regioninfo/1731033032267/Put/seqid=0 2024-11-08T02:33:43,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742388_1564 (size=7725) 2024-11-08T02:33:43,302 INFO [regionserver/ecdbff65922a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T02:33:43,451 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 934a9f648376cf09654e404de226250e 2024-11-08T02:33:43,608 INFO [regionserver/ecdbff65922a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T02:33:43,608 INFO [regionserver/ecdbff65922a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T02:33:43,651 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 934a9f648376cf09654e404de226250e 2024-11-08T02:33:43,670 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e/.tmp/info/01ff3865001248ffb41431fe7e8d9a61 2024-11-08T02:33:43,676 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/info/8f8a0e5bc87f4536847e5ba3a50faa0f 2024-11-08T02:33:43,679 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e/.tmp/info/01ff3865001248ffb41431fe7e8d9a61 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e/info/01ff3865001248ffb41431fe7e8d9a61 2024-11-08T02:33:43,683 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e/info/01ff3865001248ffb41431fe7e8d9a61, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T02:33:43,684 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 934a9f648376cf09654e404de226250e in 436ms, sequenceid=6, compaction requested=false 2024-11-08T02:33:43,688 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/namespace/934a9f648376cf09654e404de226250e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T02:33:43,688 INFO [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:33:43,688 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 934a9f648376cf09654e404de226250e: 2024-11-08T02:33:43,689 DEBUG [RS_CLOSE_REGION-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731033030989.934a9f648376cf09654e404de226250e. 2024-11-08T02:33:43,699 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/rep_barrier/8fbd7a79319c434198db700c8a9c626d is 102, key is TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9./rep_barrier:/1731033060420/DeleteFamily/seqid=0 2024-11-08T02:33:43,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742389_1565 (size=6025) 2024-11-08T02:33:43,852 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-08T02:33:44,052 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-08T02:33:44,103 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/rep_barrier/8fbd7a79319c434198db700c8a9c626d 2024-11-08T02:33:44,133 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/table/ac858f372a9547d38be326145ff58370 is 96, key is TestAcidGuarantees,,1731033032682.dc8ad63d9a6a95258392be1a735a16e9./table:/1731033060420/DeleteFamily/seqid=0 2024-11-08T02:33:44,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742390_1566 (size=5942) 2024-11-08T02:33:44,252 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1599): Waiting on 1 regions to close 2024-11-08T02:33:44,252 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T02:33:44,252 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-08T02:33:44,453 DEBUG [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-08T02:33:44,538 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.08 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/table/ac858f372a9547d38be326145ff58370 2024-11-08T02:33:44,547 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/info/8f8a0e5bc87f4536847e5ba3a50faa0f as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/info/8f8a0e5bc87f4536847e5ba3a50faa0f 2024-11-08T02:33:44,551 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/info/8f8a0e5bc87f4536847e5ba3a50faa0f, entries=22, sequenceid=93, filesize=7.5 K 2024-11-08T02:33:44,552 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/rep_barrier/8fbd7a79319c434198db700c8a9c626d as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/rep_barrier/8fbd7a79319c434198db700c8a9c626d 2024-11-08T02:33:44,555 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/rep_barrier/8fbd7a79319c434198db700c8a9c626d, entries=6, sequenceid=93, filesize=5.9 K 2024-11-08T02:33:44,556 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/.tmp/table/ac858f372a9547d38be326145ff58370 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/table/ac858f372a9547d38be326145ff58370 2024-11-08T02:33:44,559 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/table/ac858f372a9547d38be326145ff58370, entries=9, sequenceid=93, filesize=5.8 K 2024-11-08T02:33:44,560 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~20.55 KB/21040, heapSize ~35.82 KB/36680, currentSize=0 B/0 for 1588230740 in 1312ms, sequenceid=93, compaction requested=false 2024-11-08T02:33:44,563 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/data/hbase/meta/1588230740/recovered.edits/96.seqid, newMaxSeqId=96, maxSeqId=1 2024-11-08T02:33:44,564 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T02:33:44,564 INFO [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-08T02:33:44,564 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-08T02:33:44,564 DEBUG [RS_CLOSE_META-regionserver/ecdbff65922a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T02:33:44,653 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1250): stopping server ecdbff65922a,40577,1731033026611; all regions closed. 2024-11-08T02:33:44,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741834_1010 (size=26050) 2024-11-08T02:33:44,659 DEBUG [RS:0;ecdbff65922a:40577 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/oldWALs 2024-11-08T02:33:44,659 INFO [RS:0;ecdbff65922a:40577 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL ecdbff65922a%2C40577%2C1731033026611.meta:.meta(num 1731033030718) 2024-11-08T02:33:44,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741832_1008 (size=12603850) 2024-11-08T02:33:44,662 DEBUG [RS:0;ecdbff65922a:40577 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/oldWALs 2024-11-08T02:33:44,662 INFO [RS:0;ecdbff65922a:40577 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL ecdbff65922a%2C40577%2C1731033026611:(num 1731033029761) 2024-11-08T02:33:44,662 DEBUG [RS:0;ecdbff65922a:40577 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:44,662 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T02:33:44,663 INFO [RS:0;ecdbff65922a:40577 {}] hbase.ChoreService(370): Chore service for: regionserver/ecdbff65922a:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T02:33:44,663 INFO [regionserver/ecdbff65922a:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T02:33:44,663 INFO [RS:0;ecdbff65922a:40577 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40577 2024-11-08T02:33:44,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T02:33:44,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ecdbff65922a,40577,1731033026611 2024-11-08T02:33:44,711 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ecdbff65922a,40577,1731033026611] 2024-11-08T02:33:44,711 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing ecdbff65922a,40577,1731033026611; numProcessing=1 2024-11-08T02:33:44,719 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/ecdbff65922a,40577,1731033026611 already deleted, retry=false 2024-11-08T02:33:44,719 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; ecdbff65922a,40577,1731033026611 expired; onlineServers=0 2024-11-08T02:33:44,719 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'ecdbff65922a,40841,1731033025873' ***** 2024-11-08T02:33:44,719 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T02:33:44,720 DEBUG [M:0;ecdbff65922a:40841 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d237f2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ecdbff65922a/172.17.0.2:0 2024-11-08T02:33:44,720 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HRegionServer(1224): stopping server ecdbff65922a,40841,1731033025873 2024-11-08T02:33:44,720 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HRegionServer(1250): stopping server ecdbff65922a,40841,1731033025873; all regions closed. 2024-11-08T02:33:44,720 DEBUG [M:0;ecdbff65922a:40841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T02:33:44,720 DEBUG [M:0;ecdbff65922a:40841 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T02:33:44,720 DEBUG [M:0;ecdbff65922a:40841 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T02:33:44,720 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T02:33:44,720 DEBUG [master/ecdbff65922a:0:becomeActiveMaster-HFileCleaner.large.0-1731033029408 {}] cleaner.HFileCleaner(306): Exit Thread[master/ecdbff65922a:0:becomeActiveMaster-HFileCleaner.large.0-1731033029408,5,FailOnTimeoutGroup] 2024-11-08T02:33:44,720 DEBUG [master/ecdbff65922a:0:becomeActiveMaster-HFileCleaner.small.0-1731033029413 {}] cleaner.HFileCleaner(306): Exit Thread[master/ecdbff65922a:0:becomeActiveMaster-HFileCleaner.small.0-1731033029413,5,FailOnTimeoutGroup] 2024-11-08T02:33:44,720 INFO [M:0;ecdbff65922a:40841 {}] hbase.ChoreService(370): Chore service for: master/ecdbff65922a:0 had [] on shutdown 2024-11-08T02:33:44,720 DEBUG [M:0;ecdbff65922a:40841 {}] master.HMaster(1733): Stopping service threads 2024-11-08T02:33:44,720 INFO [M:0;ecdbff65922a:40841 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T02:33:44,720 ERROR [M:0;ecdbff65922a:40841 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (59733779) connection to localhost/127.0.0.1:46319 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46319,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-08T02:33:44,721 INFO [M:0;ecdbff65922a:40841 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T02:33:44,721 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T02:33:44,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T02:33:44,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T02:33:44,728 DEBUG [M:0;ecdbff65922a:40841 {}] zookeeper.ZKUtil(347): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T02:33:44,728 WARN [M:0;ecdbff65922a:40841 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T02:33:44,728 INFO [M:0;ecdbff65922a:40841 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-08T02:33:44,728 INFO [M:0;ecdbff65922a:40841 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T02:33:44,728 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T02:33:44,728 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:33:44,728 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:33:44,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T02:33:44,728 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T02:33:44,728 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:33:44,728 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=730.16 KB heapSize=895.26 KB 2024-11-08T02:33:44,745 DEBUG [M:0;ecdbff65922a:40841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e4bab31031b4c978eb0f6e97efa0d81 is 82, key is hbase:meta,,1/info:regioninfo/1731033030846/Put/seqid=0 2024-11-08T02:33:44,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742391_1567 (size=5672) 2024-11-08T02:33:44,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:33:44,811 INFO [RS:0;ecdbff65922a:40577 {}] regionserver.HRegionServer(1307): Exiting; stopping=ecdbff65922a,40577,1731033026611; zookeeper connection closed. 2024-11-08T02:33:44,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40577-0x10119916ccb0001, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:33:44,812 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@621f417b {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@621f417b 2024-11-08T02:33:44,812 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T02:33:45,149 INFO [M:0;ecdbff65922a:40841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2024 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e4bab31031b4c978eb0f6e97efa0d81 2024-11-08T02:33:45,178 DEBUG [M:0;ecdbff65922a:40841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/288c823a11f64801baf3a7207ed41584 is 2278, key is \x00\x00\x00\x00\x00\x00\x00&/proc:d/1731033063863/Put/seqid=0 2024-11-08T02:33:45,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742392_1568 (size=43687) 2024-11-08T02:33:45,583 INFO [M:0;ecdbff65922a:40841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=729.61 KB at sequenceid=2024 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/288c823a11f64801baf3a7207ed41584 2024-11-08T02:33:45,592 INFO [M:0;ecdbff65922a:40841 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 288c823a11f64801baf3a7207ed41584 2024-11-08T02:33:45,610 DEBUG [M:0;ecdbff65922a:40841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3abab8ad5be406991c9907eeab98d0a is 69, key is ecdbff65922a,40577,1731033026611/rs:state/1731033029538/Put/seqid=0 2024-11-08T02:33:45,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073742393_1569 (size=5156) 2024-11-08T02:33:46,014 INFO [M:0;ecdbff65922a:40841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2024 (bloomFilter=true), to=hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3abab8ad5be406991c9907eeab98d0a 2024-11-08T02:33:46,024 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3e4bab31031b4c978eb0f6e97efa0d81 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e4bab31031b4c978eb0f6e97efa0d81 2024-11-08T02:33:46,031 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3e4bab31031b4c978eb0f6e97efa0d81, entries=8, sequenceid=2024, filesize=5.5 K 2024-11-08T02:33:46,032 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/288c823a11f64801baf3a7207ed41584 as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/288c823a11f64801baf3a7207ed41584 2024-11-08T02:33:46,035 INFO [M:0;ecdbff65922a:40841 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 288c823a11f64801baf3a7207ed41584 2024-11-08T02:33:46,035 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/288c823a11f64801baf3a7207ed41584, entries=171, sequenceid=2024, filesize=42.7 K 2024-11-08T02:33:46,036 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3abab8ad5be406991c9907eeab98d0a as hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3abab8ad5be406991c9907eeab98d0a 2024-11-08T02:33:46,039 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46319/user/jenkins/test-data/047f2d09-47bb-4a75-bbc7-7107ec849643/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3abab8ad5be406991c9907eeab98d0a, entries=1, sequenceid=2024, filesize=5.0 K 2024-11-08T02:33:46,040 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(3040): Finished flush of dataSize ~730.16 KB/747687, heapSize ~894.96 KB/916440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1312ms, sequenceid=2024, compaction requested=false 2024-11-08T02:33:46,041 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T02:33:46,041 DEBUG [M:0;ecdbff65922a:40841 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-08T02:33:46,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39161 is added to blk_1073741830_1006 (size=880454) 2024-11-08T02:33:46,043 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-08T02:33:46,043 INFO [M:0;ecdbff65922a:40841 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-08T02:33:46,043 INFO [M:0;ecdbff65922a:40841 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:40841 2024-11-08T02:33:46,086 DEBUG [M:0;ecdbff65922a:40841 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/ecdbff65922a,40841,1731033025873 already deleted, retry=false 2024-11-08T02:33:46,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:33:46,194 INFO [M:0;ecdbff65922a:40841 {}] regionserver.HRegionServer(1307): Exiting; stopping=ecdbff65922a,40841,1731033025873; zookeeper connection closed. 2024-11-08T02:33:46,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40841-0x10119916ccb0000, quorum=127.0.0.1:56877, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T02:33:46,202 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1f79ec76{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T02:33:46,206 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@576ebda6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T02:33:46,206 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T02:33:46,206 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4727fac8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T02:33:46,206 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47db50b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/hadoop.log.dir/,STOPPED} 2024-11-08T02:33:46,208 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T02:33:46,208 WARN [BP-1683983252-172.17.0.2-1731033022817 heartbeating to localhost/127.0.0.1:46319 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T02:33:46,208 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T02:33:46,208 WARN [BP-1683983252-172.17.0.2-1731033022817 heartbeating to localhost/127.0.0.1:46319 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1683983252-172.17.0.2-1731033022817 (Datanode Uuid 29f3b9ad-8562-4ec9-b455-cdc99a1c76ce) service to localhost/127.0.0.1:46319 2024-11-08T02:33:46,211 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c/dfs/data/data1/current/BP-1683983252-172.17.0.2-1731033022817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T02:33:46,211 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/cluster_5cdd75dc-8e6e-782b-b9fb-77d1dc5f583c/dfs/data/data2/current/BP-1683983252-172.17.0.2-1731033022817 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T02:33:46,212 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T02:33:46,219 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b03fcff{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T02:33:46,220 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e0a3ea0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T02:33:46,220 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T02:33:46,220 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62a9beb2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T02:33:46,220 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8167a4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c85f3377-82fa-a36c-298a-9097634c109a/hadoop.log.dir/,STOPPED} 2024-11-08T02:33:46,234 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-08T02:33:46,350 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down